web2py.gluon.dal

Package web2py :: Package gluon :: Module dal |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.dal

       1  #!/bin/env python
       2  # -*- coding: utf-8 -*-
       3
       4  """
       5  This file is part of the web2py Web Framework
       6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
       7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
       8
       9  Thanks to
      10      * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support
      11      * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support
      12      * Denes
      13      * Chris Clark
      14      * clach05
      15      * Denes Lengyel
      16      * and many others who have contributed to current and previous versions
      17
      18  This file contains the DAL support for many relational databases,
      19  including:
      20  - SQLite
      21  - MySQL
      22  - Postgres
      23  - Oracle
      24  - MS SQL
      25  - DB2
      26  - Interbase
      27  - Ingres
      28  - SapDB (experimental)
      29  - Cubrid (experimental)
      30  - CouchDB (experimental)
      31  - MongoDB (in progress)
      32  - Google:nosql
      33  - Google:sql
      34  - Teradata
      35  - IMAP (experimental)
      36
      37  Example of usage:
      38
      39  >>> # from dal import DAL, Field
      40
      41  ### create DAL connection (and create DB if it doesn't exist)
      42  >>> db = DAL(('mysql://a:b@localhost/x', 'sqlite://storage.sqlite'), folder=None)
      43
      44  ### define a table 'person' (create/alter as necessary)
      45  >>> person = db.define_table('person',Field('name','string'))
      46
      47  ### insert a record
      48  >>> id = person.insert(name='James')
      49
      50  ### retrieve it by id
      51  >>> james = person(id)
      52
      53  ### retrieve it by name
      54  >>> james = person(name='James')
      55
      56  ### retrieve it by arbitrary query
      57  >>> query = (person.name=='James') & (person.name.startswith('J'))
      58  >>> james = db(query).select(person.ALL)[0]
      59
      60  ### update one record
      61  >>> james.update_record(name='Jim')
      62
      63  ### update multiple records by query
      64  >>> db(person.name.like('J%')).update(name='James')
      65  1
      66
      67  ### delete records by query
      68  >>> db(person.name.lower() == 'jim').delete()
      69  0
      70
      71  ### retrieve multiple records (rows)
      72  >>> people = db(person).select(orderby=person.name, groupby=person.name, limitby=(0,100))
      73
      74  ### further filter them
      75  >>> james = people.find(lambda row: row.name == 'James').first()
      76  >>> print james.id, james.name
      77  1 James
      78
      79  ### check aggregates
      80  >>> counter = person.id.count()
      81  >>> print db(person).select(counter).first()(counter)
      82  1
      83
      84  ### delete one record
      85  >>> james.delete_record()
      86  1
      87
      88  ### delete (drop) entire database table
      89  >>> person.drop()
      90
      91  Supported field types:
      92  id string text boolean integer double decimal password upload blob time date datetime
      93
      94  Supported DAL URI strings:
      95  'sqlite://test.db'
      96  'sqlite:memory'
      97  'jdbc:sqlite://test.db'
      98  'mysql://root:none@localhost/test'
      99  'postgres://mdipierro:password@localhost/test'
     100  'postgres:psycopg2://mdipierro:password@localhost/test'
     101  'postgres:pg8000://mdipierro:password@localhost/test'
     102  'jdbc:postgres://mdipierro:none@localhost/test'
     103  'mssql://web2py:none@A64X2/web2py_test'
     104  'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings
     105  'oracle://username:password@database'
     106  'firebird://user:password@server:3050/database'
     107  'db2://DSN=dsn;UID=user;PWD=pass'
     108  'firebird://username:password@hostname/database'
     109  'firebird_embedded://username:password@c://path'
     110  'informix://user:password@server:3050/database'
     111  'informixu://user:password@server:3050/database' # unicode informix
     112  'google:datastore' # for google app engine datastore
     113  'google:sql' # for google app engine with sql (mysql compatible)
     114  'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
     115  'imap://user:password@server:port' # experimental
     116
     117  For more info:
     118  help(DAL)
     119  help(Field)
     120  """
     121
     122  ###################################################################################
     123  # this file only exposes DAL and Field
     124  ###################################################################################
     125
     126  __all__ = ['DAL', 'Field']
     127
     128  MAXCHARLENGTH = 2**15 # not quite but reasonable default max char length
     129  DEFAULTLENGTH = {'string':512,
     130                   'password':512,
     131                   'upload':512,
     132                   'text':2**15,
     133                   'blob':2**31}
     134  TIMINGSSIZE = 100
     135
     136  import re
     137  import sys
     138  import locale
     139  import os
     140  import types
     141  import cPickle
     142  import datetime
     143  import threading
     144  import time
     145  import cStringIO
     146  import csv
     147  import cgi
     148  import copy
     149  import socket
     150  import logging
     151  import copy_reg
     152  import base64
     153  import shutil
     154  import marshal
     155  import decimal
     156  import struct
     157  import urllib
     158  import hashlib
     159  import uuid
     160  import glob
     161  import traceback
     162
     163  CALLABLETYPES = (types.LambdaType, types.FunctionType, types.BuiltinFunctionType,
     164                   types.MethodType, types.BuiltinMethodType)
     165
     166
     167  ###################################################################################
     168  # following checks allows running of dal without web2py as a standalone module
     169  ###################################################################################
     170  try:
     171      from utils import web2py_uuid
     172  except ImportError:
     173      import uuid
     174 -    def web2py_uuid(): return str(uuid.uuid4())
     175
     176  try:
     177      import portalocker
     178      have_portalocker = True
     179  except ImportError:
     180      have_portalocker = False
     181
     182  try:
     183      import serializers
     184      have_serializers = True
     185  except ImportError:
     186      have_serializers = False
     187
     188  try:
     189      import validators
     190      have_validators = True
     191  except ImportError:
     192      have_validators = False
     193
     194  logger = logging.getLogger("web2py.dal")
     195  DEFAULT = lambda:0
     196
     197  sql_locker = threading.RLock()
     198  thread = threading.local()
     199
     200  # internal representation of tables with field
     201  #  <table>.<field>, tables and fields may only be [a-zA-Z0-0_]
     202
     203  regex_type = re.compile('^([\w\_\:]+)')
     204  regex_dbname = re.compile('^(\w+)(\:\w+)*')
     205  regex_table_field = re.compile('^([\w_]+)\.([\w_]+)$')
     206  regex_content = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)\.(?P<name>\w+)\.\w+$')
     207  regex_cleanup_fn = re.compile('[\'"\s;]+')
     208  string_unpack=re.compile('(?<!\|)\|(?!\|)')
     209  regex_python_keywords = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
     210  regex_select_as_parser = re.compile("\s+AS\s+(\S+)")
     211
     212  # list of drivers will be built on the fly
     213  # and lists only what is available
     214  drivers = []
     215
     216  try:
     217      from new import classobj
     218      from google.appengine.ext import db as gae
     219      from google.appengine.api import namespace_manager, rdbms
     220      from google.appengine.api.datastore_types import Key  ### needed for belongs on ID
     221      from google.appengine.ext.db.polymodel import PolyModel
     222      drivers.append('google')
     223  except ImportError:
     224      pass
     225
     226  if not 'google' in drivers:
     227
     228      try:
     229          from pysqlite2 import dbapi2 as sqlite3
     230          drivers.append('pysqlite2')
     231      except ImportError:
     232          try:
     233              from sqlite3 import dbapi2 as sqlite3
     234              drivers.append('SQLite3')
     235          except ImportError:
     236              logger.debug('no sqlite3 or pysqlite2.dbapi2 driver')
     237
     238      try:
     239          import contrib.pymysql as pymysql
     240          drivers.append('pymysql')
     241      except ImportError:
     242          logger.debug('no pymysql driver')
     243
     244      try:
     245          import psycopg2
     246          from psycopg2.extensions import adapt as psycopg2_adapt
     247          drivers.append('psycopg2')
     248      except ImportError:
     249          logger.debug('no psycopg2 driver')
     250
     251      try:
     252          # first try contrib driver, then from site-packages (if installed)
     253          try:
     254              import contrib.pg8000.dbapi as pg8000
     255          except ImportError:
     256              import pg8000.dbapi as pg8000
     257          drivers.append('pg8000')
     258      except ImportError:
     259          logger.debug('no pg8000 driver')
     260
     261      try:
     262          import cx_Oracle
     263          drivers.append('Oracle')
     264      except ImportError:
     265          logger.debug('no cx_Oracle driver')
     266
     267      try:
     268          import pyodbc
     269          drivers.append('MSSQL/DB2/Teradata')
     270      except ImportError:
     271          logger.debug('no MSSQL/DB2/Teradata driver')
     272
     273      try:
     274          import kinterbasdb
     275          drivers.append('Interbase')
     276      except ImportError:
     277          logger.debug('no kinterbasdb driver')
     278
     279      try:
     280          import firebirdsql
     281          drivers.append('Firebird')
     282      except ImportError:
     283          logger.debug('no Firebird driver')
     284
     285      try:
     286          import informixdb
     287          drivers.append('Informix')
     288          logger.warning('Informix support is experimental')
     289      except ImportError:
     290          logger.debug('no informixdb driver')
     291
     292      try:
     293          import sapdb
     294          drivers.append('SAPDB')
     295          logger.warning('SAPDB support is experimental')
     296      except ImportError:
     297          logger.debug('no sapdb driver')
     298
     299      try:
     300          import cubriddb
     301          drivers.append('Cubrid')
     302          logger.warning('Cubrid support is experimental')
     303      except ImportError:
     304          logger.debug('no cubriddb driver')
     305
     306      try:
     307          from com.ziclix.python.sql import zxJDBC
     308          import java.sql
     309          # Try sqlite jdbc driver from http://www.zentus.com/sqlitejdbc/
     310          from org.sqlite import JDBC # required by java.sql; ensure we have it
     311          drivers.append('zxJDBC')
     312          logger.warning('zxJDBC support is experimental')
     313          is_jdbc = True
     314      except ImportError:
     315          logger.debug('no zxJDBC driver')
     316          is_jdbc = False
     317
     318      try:
     319          import ingresdbi
     320          drivers.append('Ingres')
     321      except ImportError:
     322          logger.debug('no Ingres driver')
     323      # NOTE could try JDBC.......
     324
     325      try:
     326          import couchdb
     327          drivers.append('CouchDB')
     328      except ImportError:
     329          logger.debug('no couchdb driver')
     330
     331      try:
     332          import pymongo
     333          drivers.append('mongoDB')
     334      except:
     335          logger.debug('no mongoDB driver')
     336
     337      try:
     338          import imaplib
     339          drivers.append('IMAP')
     340      except:
     341          logger.debug('could not import imaplib')
     342
     343  PLURALIZE_RULES = [
     344      (re.compile('child$'), re.compile('child$'), 'children'),
     345      (re.compile('oot$'), re.compile('oot$'), 'eet'),
     346      (re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
     347      (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
     348      (re.compile('sis$'), re.compile('sis$'), 'ses'),
     349      (re.compile('man$'), re.compile('man$'), 'men'),
     350      (re.compile('ife$'), re.compile('ife$'), 'ives'),
     351      (re.compile('eau$'), re.compile('eau$'), 'eaux'),
     352      (re.compile('lf$'), re.compile('lf$'), 'lves'),
     353      (re.compile('[sxz]$'), re.compile('$'), 'es'),
     354      (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
     355      (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
     356      (re.compile('$'), re.compile('$'), 's'),
     357      ]
     358
     359 -def pluralize(singular, rules=PLURALIZE_RULES):
     360      for line in rules:
     361          re_search, re_sub, replace = line
     362          plural = re_search.search(singular) and re_sub.sub(replace, singular)
     363          if plural: return plural
     364
     365 -def OR(a,b):
     366      return a|b
     367
     368 -def AND(a,b):
     369      return a&b
     370
     371  if 'google' in drivers:
     372
     373      is_jdbc = False
     374
     375 -    class GAEDecimalProperty(gae.Property):
     376          """
     377          GAE decimal implementation
     378          """
     379          data_type = decimal.Decimal
     380
     381 -        def __init__(self, precision, scale, **kwargs):
     382              super(GAEDecimalProperty, self).__init__(self, **kwargs)
     383              d = '1.'
     384              for x in range(scale):
     385                  d += '0'
     386              self.round = decimal.Decimal(d)
     387
     388 -        def get_value_for_datastore(self, model_instance):
     389              value = super(GAEDecimalProperty, self).get_value_for_datastore(model_instance)
     390              if value is None or value == '':
     391                  return None
     392              else:
     393                  return str(value)
     394
     395 -        def make_value_from_datastore(self, value):
     396              if value is None or value == '':
     397                  return None
     398              else:
     399                  return decimal.Decimal(value).quantize(self.round)
     400
     401 -        def validate(self, value):
     402              value = super(GAEDecimalProperty, self).validate(value)
     403              if value is None or isinstance(value, decimal.Decimal):
     404                  return value
     405              elif isinstance(value, basestring):
     406                  return decimal.Decimal(value)
     407              raise gae.BadValueError("Property %s must be a Decimal or string." % self.name)
     408
     409  ###################################################################################
     410  # class that handles connection pooling (all adapters are derived from this one)
     411  ###################################################################################
     412
     413 -class ConnectionPool(object):
     414
     415      pools = {}
     416      check_active_connection = True
     417
     418      @staticmethod
     419 -    def set_folder(folder):
     420          thread.folder = folder
     421
     422      # ## this allows gluon to commit/rollback all dbs in this thread
     423
     424      @staticmethod
     425 -    def close_all_instances(action):
     426          """ to close cleanly databases in a multithreaded environment """
     427          if not hasattr(thread, 'instances'):
     428              return
     429          while thread.instances:
     430              instance = thread.instances.pop()
     431              if action:
     432                  getattr(instance, action)()
     433              # ## if you want pools, recycle this connection
     434              really = True
     435              if instance.pool_size:
     436                  sql_locker.acquire()
     437                  pool = ConnectionPool.pools[instance.uri]
     438                  if len(pool) < instance.pool_size:
     439                      pool.append(instance.connection)
     440                      really = False
     441                  sql_locker.release()
     442              if really:
     443                  getattr(instance, 'close')()
     444          return
     445
     446 -    def find_or_make_work_folder(self):
     447          """ this actually does not make the folder. it has to be there """
     448          if hasattr(thread,'folder'):
     449              self.folder = thread.folder
     450          else:
     451              self.folder = thread.folder = ''
     452
     453          # Creating the folder if it does not exist
     454          if False and self.folder and not os.path.exists(self.folder):
     455              os.mkdir(self.folder)
     456
     457 -    def pool_connection(self, f, cursor=True):
     458          """
     459          this function defines: self.connection and self.cursor (iff cursor is True)
     460          if self.pool_size>0 it will try pull the connection from the pool
     461          if the connection is not active (closed by db server) it will loop
     462          if not self.pool_size or no active connections in pool makes a new one
     463          """
     464          if not self.pool_size:
     465              self.connection = f()
     466              self.cursor = cursor and self.connection.cursor()
     467          else:
     468              uri = self.uri
     469              while True:
     470                  sql_locker.acquire()
     471                  if not uri in ConnectionPool.pools:
     472                      ConnectionPool.pools[uri] = []
     473                  if ConnectionPool.pools[uri]:
     474                      self.connection = ConnectionPool.pools[uri].pop()
     475                      sql_locker.release()
     476                      self.cursor = cursor and self.connection.cursor()
     477                      try:
     478                          if self.cursor and self.check_active_connection:
     479                              self.execute('SELECT 1;')
     480                          break
     481                      except:
     482                          pass
     483                  else:
     484                      sql_locker.release()
     485                      self.connection = f()
     486                      self.cursor = cursor and self.connection.cursor()
     487                      break
     488          if not hasattr(thread,'instances'):
     489              thread.instances = []
     490          thread.instances.append(self)
     491
     492
     493  ###################################################################################
     494  # this is a generic adapter that does nothing; all others are derived from this one
     495  ###################################################################################
     496
     497 -class BaseAdapter(ConnectionPool):
     498
     499      driver = None
     500      maxcharlength = MAXCHARLENGTH
     501      commit_on_alter_table = False
     502      support_distributed_transaction = False
     503      uploads_in_blob = False
     504      can_select_for_update = True
     505      types = {
     506          'boolean': 'CHAR(1)',
     507          'string': 'CHAR(%(length)s)',
     508          'text': 'TEXT',
     509          'password': 'CHAR(%(length)s)',
     510          'blob': 'BLOB',
     511          'upload': 'CHAR(%(length)s)',
     512          'integer': 'INTEGER',
     513          'double': 'DOUBLE',
     514          'decimal': 'DOUBLE',
     515          'date': 'DATE',
     516          'time': 'TIME',
     517          'datetime': 'TIMESTAMP',
     518          'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
     519          'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
     520          'list:integer': 'TEXT',
     521          'list:string': 'TEXT',
     522          'list:reference': 'TEXT',
     523          }
     524
     525 -    def adapt(self, obj):
     526          return "'%s'" % obj.replace("'", "''")
     527
     528 -    def integrity_error(self):
     529          return self.driver.IntegrityError
     530
     531 -    def operational_error(self):
     532          return self.driver.OperationalError
     533
     534 -    def file_exists(self, filename):
     535          """
     536          to be used ONLY for files that on GAE may not be on filesystem
     537          """
     538          return os.path.exists(filename)
     539
     540 -    def file_open(self, filename, mode='rb', lock=True):
     541          """
     542          to be used ONLY for files that on GAE may not be on filesystem
     543          """
     544          if have_portalocker and lock:
     545              fileobj = portalocker.LockedFile(filename,mode)
     546          else:
     547              fileobj = open(filename,mode)
     548          return fileobj
     549
     550 -    def file_close(self, fileobj):
     551          """
     552          to be used ONLY for files that on GAE may not be on filesystem
     553          """
     554          if fileobj:
     555              fileobj.close()
     556
     557 -    def file_delete(self, filename):
     558          os.unlink(filename)
     559
     560 -    def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
     561                   credential_decoder=lambda x:x, driver_args={},
     562                   adapter_args={}):
     563          self.db = db
     564          self.dbengine = "None"
     565          self.uri = uri
     566          self.pool_size = pool_size
     567          self.folder = folder
     568          self.db_codec = db_codec
     569          class Dummy(object):
     570              lastrowid = 1
     571              def __getattr__(self, value):
     572                  return lambda *a, **b: []
     573          self.connection = Dummy()
     574          self.cursor = Dummy()
     575
     576 -    def sequence_name(self,tablename):
     577          return '%s_sequence' % tablename
     578
     579 -    def trigger_name(self,tablename):
     580          return '%s_sequence' % tablename
     581
     582 -    def create_table(self, table,
     583                       migrate=True,
     584                       fake_migrate=False,
     585                       polymodel=None):
     586          fields = []
     587          sql_fields = {}
     588          sql_fields_aux = {}
     589          TFK = {}
     590          tablename = table._tablename
     591          sortable = 0
     592          for field in table:
     593              sortable += 1
     594              k = field.name
     595              if isinstance(field.type,SQLCustomType):
     596                  ftype = field.type.native or field.type.type
     597              elif field.type.startswith('reference'):
     598                  referenced = field.type[10:].strip()
     599                  constraint_name = self.constraint_name(tablename, field.name)
     600                  if hasattr(table,'_primarykey'):
     601                      rtablename,rfieldname = referenced.split('.')
     602                      rtable = table._db[rtablename]
     603                      rfield = rtable[rfieldname]
     604                      # must be PK reference or unique
     605                      if rfieldname in rtable._primarykey or rfield.unique:
     606                          ftype = self.types[rfield.type[:9]] % dict(length=rfield.length)
     607                          # multicolumn primary key reference?
     608                          if not rfield.unique and len(rtable._primarykey)>1 :
     609                              # then it has to be a table level FK
     610                              if rtablename not in TFK:
     611                                  TFK[rtablename] = {}
     612                              TFK[rtablename][rfieldname] = field.name
     613                          else:
     614                              ftype = ftype + \
     615                                  self.types['reference FK'] %dict(\
     616                                  constraint_name=constraint_name,
     617                                  table_name=tablename,
     618                                  field_name=field.name,
     619                                  foreign_key='%s (%s)'%(rtablename, rfieldname),
     620                                  on_delete_action=field.ondelete)
     621                  else:
     622                      # make a guess here for circular references
     623                      id_fieldname = referenced in table._db and table._db[referenced]._id.name or 'id'
     624                      ftype = self.types[field.type[:9]]\
     625                          % dict(table_name=tablename,
     626                                 field_name=field.name,
     627                                 constraint_name=constraint_name,
     628                                 foreign_key=referenced + ('(%s)' % id_fieldname),
     629                                 on_delete_action=field.ondelete)
     630              elif field.type.startswith('list:reference'):
     631                  ftype = self.types[field.type[:14]]
     632              elif field.type.startswith('decimal'):
     633                  precision, scale = map(int,field.type[8:-1].split(','))
     634                  ftype = self.types[field.type[:7]] % \
     635                      dict(precision=precision,scale=scale)
     636              elif not field.type in self.types:
     637                  raise SyntaxError, 'Field: unknown field type: %s for %s' % \
     638                      (field.type, field.name)
     639              else:
     640                  ftype = self.types[field.type]\
     641                       % dict(length=field.length)
     642              if not field.type.startswith('id') and not field.type.startswith('reference'):
     643                  if field.notnull:
     644                      ftype += ' NOT NULL'
     645                  else:
     646                      ftype += self.ALLOW_NULL()
     647                  if field.unique:
     648                      ftype += ' UNIQUE'
     649
     650              # add to list of fields
     651              sql_fields[field.name] = dict(sortable=sortable,
     652                                            type=str(field.type),
     653                                            sql=ftype)
     654
     655              if isinstance(field.default,(str,int,float)):
     656                  # Caveat: sql_fields and sql_fields_aux differ for default values.
     657                  # sql_fields is used to trigger migrations and sql_fields_aux
     658                  # is used for create tables.
     659                  # The reason is that we do not want to trigger a migration simply
     660                  # because a default value changes.
     661                  not_null = self.NOT_NULL(field.default, field.type)
     662                  ftype = ftype.replace('NOT NULL', not_null)
     663              sql_fields_aux[field.name] = dict(sql=ftype)
     664              fields.append('%s %s' % (field.name, ftype))
     665          other = ';'
     666
     667          # backend-specific extensions to fields
     668          if self.dbengine == 'mysql':
     669              if not hasattr(table, "_primarykey"):
     670                  fields.append('PRIMARY KEY(%s)' % table._id.name)
     671              other = ' ENGINE=InnoDB CHARACTER SET utf8;'
     672
     673          fields = ',\n    '.join(fields)
     674          for rtablename in TFK:
     675              rfields = TFK[rtablename]
     676              pkeys = table._db[rtablename]._primarykey
     677              fkeys = [ rfields[k] for k in pkeys ]
     678              fields = fields + ',\n    ' + \
     679                       self.types['reference TFK'] %\
     680                       dict(table_name=tablename,
     681                       field_name=', '.join(fkeys),
     682                       foreign_table=rtablename,
     683                       foreign_key=', '.join(pkeys),
     684                       on_delete_action=field.ondelete)
     685
     686          if hasattr(table,'_primarykey'):
     687              query = '''CREATE TABLE %s(\n    %s,\n    %s) %s''' % \
     688                  (tablename, fields, self.PRIMARY_KEY(', '.join(table._primarykey)),other)
     689          else:
     690              query = '''CREATE TABLE %s(\n    %s\n)%s''' % \
     691                  (tablename, fields, other)
     692
     693          if self.uri.startswith('sqlite:///'):
     694              path_encoding = sys.getfilesystemencoding() or locale.getdefaultlocale()[1] or 'utf8'
     695              dbpath = self.uri[9:self.uri.rfind('/')].decode('utf8').encode(path_encoding)
     696          else:
     697              dbpath = self.folder
     698
     699          if not migrate:
     700              return query
     701          elif self.uri.startswith('sqlite:memory'):
     702              table._dbt = None
     703          elif isinstance(migrate, str):
     704              table._dbt = os.path.join(dbpath, migrate)
     705          else:
     706              table._dbt = os.path.join(dbpath, '%s_%s.table' \
     707                                            % (table._db._uri_hash, tablename))
     708          if table._dbt:
     709              table._loggername = os.path.join(dbpath, 'sql.log')
     710              logfile = self.file_open(table._loggername, 'a')
     711          else:
     712              logfile = None
     713          if not table._dbt or not self.file_exists(table._dbt):
     714              if table._dbt:
     715                  logfile.write('timestamp: %s\n'
     716                                 % datetime.datetime.today().isoformat())
     717                  logfile.write(query + '\n')
     718              if not fake_migrate:
     719                  self.create_sequence_and_triggers(query,table)
     720                  table._db.commit()
     721              if table._dbt:
     722                  tfile = self.file_open(table._dbt, 'w')
     723                  cPickle.dump(sql_fields, tfile)
     724                  self.file_close(tfile)
     725                  if fake_migrate:
     726                      logfile.write('faked!\n')
     727                  else:
     728                      logfile.write('success!\n')
     729          else:
     730              tfile = self.file_open(table._dbt, 'r')
     731              try:
     732                  sql_fields_old = cPickle.load(tfile)
     733              except EOFError:
     734                  self.file_close(tfile)
     735                  self.file_close(logfile)
     736                  raise RuntimeError, 'File %s appears corrupted' % table._dbt
     737              self.file_close(tfile)
     738              if sql_fields != sql_fields_old:
     739                  self.migrate_table(table,
     740                                     sql_fields, sql_fields_old,
     741                                     sql_fields_aux, logfile,
     742                                     fake_migrate=fake_migrate)
     743          self.file_close(logfile)
     744          return query
     745
     746 -    def migrate_table(
     747          self,
     748          table,
     749          sql_fields,
     750          sql_fields_old,
     751          sql_fields_aux,
     752          logfile,
     753          fake_migrate=False,
     754          ):
     755          tablename = table._tablename
     756          def fix(item):
     757              k,v=item
     758              if not isinstance(v,dict):
     759                  v=dict(type='unkown',sql=v)
     760              return k.lower(),v
     761          ### make sure all field names are lower case to avoid conflicts
     762          sql_fields = dict(map(fix,sql_fields.items()))
     763          sql_fields_old = dict(map(fix,sql_fields_old.items()))
     764          sql_fields_aux = dict(map(fix,sql_fields_aux.items()))
     765
     766          keys = sql_fields.keys()
     767          for key in sql_fields_old:
     768              if not key in keys:
     769                  keys.append(key)
     770          if self.dbengine == 'mssql':
     771              new_add = '; ALTER TABLE %s ADD ' % tablename
     772          else:
     773              new_add = ', ADD '
     774
     775          metadata_change = False
     776          sql_fields_current = copy.copy(sql_fields_old)
     777          for key in keys:
     778              query = None
     779              if not key in sql_fields_old:
     780                  sql_fields_current[key] = sql_fields[key]
     781                  query = ['ALTER TABLE %s ADD %s %s;' % \
     782                           (tablename, key,
     783                            sql_fields_aux[key]['sql'].replace(', ', new_add))]
     784                  metadata_change = True
     785              elif self.dbengine == 'sqlite':
     786                  if key in sql_fields:
     787                      sql_fields_current[key] = sql_fields[key]
     788                  metadata_change = True
     789              elif not key in sql_fields:
     790                  del sql_fields_current[key]
     791                  if not self.dbengine in ('firebird',):
     792                      query = ['ALTER TABLE %s DROP COLUMN %s;' % (tablename, key)]
     793                  else:
     794                      query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
     795                  metadata_change = True
     796              elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
     797                    and not isinstance(table[key].type, SQLCustomType) \
     798                    and not (table[key].type.startswith('reference') and \
     799                        sql_fields[key]['sql'].startswith('INT,') and \
     800                        sql_fields_old[key]['sql'].startswith('INT NOT NULL,')):
     801                  sql_fields_current[key] = sql_fields[key]
     802                  t = tablename
     803                  tt = sql_fields_aux[key]['sql'].replace(', ', new_add)
     804                  if not self.dbengine in ('firebird',):
     805                      query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
     806                               'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
     807                               'ALTER TABLE %s DROP COLUMN %s;' % (t, key),
     808                               'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
     809                               'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
     810                               'ALTER TABLE %s DROP COLUMN %s__tmp;' % (t, key)]
     811                  else:
     812                      query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
     813                               'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
     814                               'ALTER TABLE %s DROP %s;' % (t, key),
     815                               'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
     816                               'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
     817                               'ALTER TABLE %s DROP %s__tmp;' % (t, key)]
     818                  metadata_change = True
     819              elif sql_fields[key]['type'] != sql_fields_old[key]['type']:
     820                  sql_fields_current[key] = sql_fields[key]
     821                  metadata_change = True
     822
     823              if query:
     824                  logfile.write('timestamp: %s\n'
     825                                % datetime.datetime.today().isoformat())
     826                  table._db['_lastsql'] = '\n'.join(query)
     827                  for sub_query in query:
     828                      logfile.write(sub_query + '\n')
     829                      if not fake_migrate:
     830                          self.execute(sub_query)
     831                          # Caveat: mysql, oracle and firebird do not allow multiple alter table
     832                          # in one transaction so we must commit partial transactions and
     833                          # update table._dbt after alter table.
     834                          if table._db._adapter.commit_on_alter_table:
     835                              table._db.commit()
     836                              tfile = self.file_open(table._dbt, 'w')
     837                              cPickle.dump(sql_fields_current, tfile)
     838                              self.file_close(tfile)
     839                              logfile.write('success!\n')
     840                      else:
     841                          logfile.write('faked!\n')
     842              elif metadata_change:
     843                  tfile = self.file_open(table._dbt, 'w')
     844                  cPickle.dump(sql_fields_current, tfile)
     845                  self.file_close(tfile)
     846
     847          if metadata_change and \
     848                  not (query and self.dbengine in ('mysql','oracle','firebird')):
     849              table._db.commit()
     850              tfile = self.file_open(table._dbt, 'w')
     851              cPickle.dump(sql_fields_current, tfile)
     852              self.file_close(tfile)
     853
     854 -    def LOWER(self, first):
     855          return 'LOWER(%s)' % self.expand(first)
     856
     857 -    def UPPER(self, first):
     858          return 'UPPER(%s)' % self.expand(first)
     859
     860 -    def EXTRACT(self, first, what):
     861          return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
     862
     863 -    def AGGREGATE(self, first, what):
     864          return "%s(%s)" % (what, self.expand(first))
     865
     866 -    def JOIN(self):
     867          return 'JOIN'
     868
     869 -    def LEFT_JOIN(self):
     870          return 'LEFT JOIN'
     871
     872 -    def RANDOM(self):
     873          return 'Random()'
     874
     875 -    def NOT_NULL(self, default, field_type):
     876          return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
     877
     878 -    def COALESCE(self, first, second):
     879          expressions = [self.expand(first)]+[self.expand(e) for e in second]
     880          return 'COALESCE(%s)' % ','.join(expressions)
     881
     882 -    def COALESCE_ZERO(self, first):
     883          return 'COALESCE(%s,0)' % self.expand(first)
     884
     885 -    def RAW(self, first):
     886          return first
     887
     888 -    def ALLOW_NULL(self):
     889          return ''
     890
     891 -    def SUBSTRING(self, field, parameters):
     892          return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
     893
     894 -    def PRIMARY_KEY(self, key):
     895          return 'PRIMARY KEY(%s)' % key
     896
     897 -    def _drop(self, table, mode):
     898          return ['DROP TABLE %s;' % table]
     899
     900 -    def drop(self, table, mode=''):
     901          if table._dbt:
     902              logfile = self.file_open(table._loggername, 'a')
     903          queries = self._drop(table, mode)
     904          for query in queries:
     905              if table._dbt:
     906                  logfile.write(query + '\n')
     907              self.execute(query)
     908          table._db.commit()
     909          del table._db[table._tablename]
     910          del table._db.tables[table._db.tables.index(table._tablename)]
     911          table._db._update_referenced_by(table._tablename)
     912          if table._dbt:
     913              self.file_delete(table._dbt)
     914              logfile.write('success!\n')
     915
     916 -    def _insert(self, table, fields):
     917          keys = ','.join(f.name for f,v in fields)
     918          values = ','.join(self.expand(v,f.type) for f,v in fields)
     919          return 'INSERT INTO %s(%s) VALUES (%s);' % (table, keys, values)
     920
     921 -    def insert(self, table, fields):
     922          query = self._insert(table,fields)
     923          try:
     924              self.execute(query)
     925          except Exception, e:
     926              if isinstance(e,self.integrity_error_class()):
     927                  return None
     928              raise e
     929          if hasattr(table,'_primarykey'):
     930              return dict([(k[0].name, k[1]) for k in fields \
     931                               if k[0].name in table._primarykey])
     932          id = self.lastrowid(table)
     933          if not isinstance(id,int):
     934              return id
     935          rid = Reference(id)
     936          (rid._table, rid._record) = (table, None)
     937          return rid
     938
     939 -    def bulk_insert(self, table, items):
     940          return [self.insert(table,item) for item in items]
     941
     942 -    def NOT(self, first):
     943          return '(NOT %s)' % self.expand(first)
     944
     945 -    def AND(self, first, second):
     946          return '(%s AND %s)' % (self.expand(first), self.expand(second))
     947
     948 -    def OR(self, first, second):
     949          return '(%s OR %s)' % (self.expand(first), self.expand(second))
     950
     951 -    def BELONGS(self, first, second):
     952          if isinstance(second, str):
     953              return '(%s IN (%s))' % (self.expand(first), second[:-1])
     954          elif second==[] or second==():
     955              return '(1=0)'
     956          items = ','.join(self.expand(item, first.type) for item in second)
     957          return '(%s IN (%s))' % (self.expand(first), items)
     958
     959 -    def REGEXP(self, first, second):
     960          "regular expression operator"
     961          raise NotImplementedError
     962
     963 -    def LIKE(self, first, second):
     964          "case sensitive like operator"
     965          raise NotImplementedError
     966
     967 -    def ILIKE(self, first, second):
     968          "case in-sensitive like operator"
     969          return '(%s LIKE %s)' % (self.expand(first), self.expand(second, 'string'))
     970
     971 -    def STARTSWITH(self, first, second):
     972          return '(%s LIKE %s)' % (self.expand(first), self.expand(second+'%', 'string'))
     973
     974 -    def ENDSWITH(self, first, second):
     975          return '(%s LIKE %s)' % (self.expand(first), self.expand('%'+second, 'string'))
     976
     977 -    def CONTAINS(self, first, second):
     978          if first.type in ('string', 'text'):
     979              key = '%'+str(second).replace('%','%%')+'%'
     980          elif first.type.startswith('list:'):
     981              key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%'
     982          return '(%s LIKE %s)' % (self.expand(first),self.expand(key,'string'))
     983
     984 -    def EQ(self, first, second=None):
     985          if second is None:
     986              return '(%s IS NULL)' % self.expand(first)
     987          return '(%s = %s)' % (self.expand(first), self.expand(second, first.type))
     988
     989 -    def NE(self, first, second=None):
     990          if second is None:
     991              return '(%s IS NOT NULL)' % self.expand(first)
     992          return '(%s <> %s)' % (self.expand(first), self.expand(second, first.type))
     993
     994 -    def LT(self,first,second=None):
     995          if second is None:
     996              raise RuntimeError, "Cannot compare %s < None" % first
     997          return '(%s < %s)' % (self.expand(first),self.expand(second,first.type))
     998
     999 -    def LE(self,first,second=None):
    1000          if second is None:
    1001              raise RuntimeError, "Cannot compare %s <= None" % first
    1002          return '(%s <= %s)' % (self.expand(first),self.expand(second,first.type))
    1003
    1004 -    def GT(self,first,second=None):
    1005          if second is None:
    1006              raise RuntimeError, "Cannot compare %s > None" % first
    1007          return '(%s > %s)' % (self.expand(first),self.expand(second,first.type))
    1008
    1009 -    def GE(self,first,second=None):
    1010          if second is None:
    1011              raise RuntimeError, "Cannot compare %s >= None" % first
    1012          return '(%s >= %s)' % (self.expand(first),self.expand(second,first.type))
    1013
    1014 -    def ADD(self, first, second):
    1015          return '(%s + %s)' % (self.expand(first), self.expand(second, first.type))
    1016
    1017 -    def SUB(self, first, second):
    1018          return '(%s - %s)' % (self.expand(first), self.expand(second, first.type))
    1019
    1020 -    def MUL(self, first, second):
    1021          return '(%s * %s)' % (self.expand(first), self.expand(second, first.type))
    1022
    1023 -    def DIV(self, first, second):
    1024          return '(%s / %s)' % (self.expand(first), self.expand(second, first.type))
    1025
    1026 -    def MOD(self, first, second):
    1027          return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type))
    1028
    1029 -    def AS(self, first, second):
    1030          return '%s AS %s' % (self.expand(first), second)
    1031
    1032 -    def ON(self, first, second):
    1033          return '%s ON %s' % (self.expand(first), self.expand(second))
    1034
    1035 -    def INVERT(self, first):
    1036          return '%s DESC' % self.expand(first)
    1037
    1038 -    def COMMA(self, first, second):
    1039          return '%s, %s' % (self.expand(first), self.expand(second))
    1040
    1041 -    def expand(self, expression, field_type=None):
    1042          if isinstance(expression, Field):
    1043              return str(expression)
    1044          elif isinstance(expression, (Expression, Query)):
    1045              if not expression.second is None:
    1046                  return expression.op(expression.first, expression.second)
    1047              elif not expression.first is None:
    1048                  return expression.op(expression.first)
    1049              elif not isinstance(expression.op, str):
    1050                  return expression.op()
    1051              else:
    1052                  return '(%s)' % expression.op
    1053          elif field_type:
    1054              return str(self.represent(expression,field_type))
    1055          elif isinstance(expression,(list,tuple)):
    1056              return ','.join(self.represent(item,field_type) for item in expression)
    1057          else:
    1058              return str(expression)
    1059
    1060 -    def alias(self, table, alias):
    1061          """
    1062          Given a table object, makes a new table object
    1063          with alias name.
    1064          """
    1065          other = copy.copy(table)
    1066          other['_ot'] = other._tablename
    1067          other['ALL'] = SQLALL(other)
    1068          other['_tablename'] = alias
    1069          for fieldname in other.fields:
    1070              other[fieldname] = copy.copy(other[fieldname])
    1071              other[fieldname]._tablename = alias
    1072              other[fieldname].tablename = alias
    1073              other[fieldname].table = other
    1074          table._db[alias] = other
    1075          return other
    1076
    1077 -    def _truncate(self, table, mode=''):
    1078          tablename = table._tablename
    1079          return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
    1080
    1081 -    def truncate(self, table, mode= ' '):
    1082          # Prepare functions "write_to_logfile" and "close_logfile"
    1083          if table._dbt:
    1084              logfile = self.file_open(table._loggername, 'a')
    1085          else:
    1086              class Logfile(object):
    1087                  def write(self, value):
    1088                      pass
    1089                  def close(self):
    1090                      pass
    1091              logfile = Logfile()
    1092
    1093          try:
    1094              queries = table._db._adapter._truncate(table, mode)
    1095              for query in queries:
    1096                  logfile.write(query + '\n')
    1097                  self.execute(query)
    1098              table._db.commit()
    1099              logfile.write('success!\n')
    1100          finally:
    1101              logfile.close()
    1102
    1103 -    def _update(self, tablename, query, fields):
    1104          if query:
    1105              if use_common_filters(query):
    1106                  query = self.common_filter(query, [tablename])
    1107              sql_w = ' WHERE ' + self.expand(query)
    1108          else:
    1109              sql_w = ''
    1110          sql_v = ','.join(['%s=%s' % (field.name, self.expand(value, field.type)) \
    1111                                for (field, value) in fields])
    1112          return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
    1113
    1114 -    def update(self, tablename, query, fields):
    1115          sql = self._update(tablename, query, fields)
    1116          self.execute(sql)
    1117          try:
    1118              return self.cursor.rowcount
    1119          except:
    1120              return None
    1121
    1122 -    def _delete(self, tablename, query):
    1123          if query:
    1124              if use_common_filters(query):
    1125                  query = self.common_filter(query, [tablename])
    1126              sql_w = ' WHERE ' + self.expand(query)
    1127          else:
    1128              sql_w = ''
    1129          return 'DELETE FROM %s%s;' % (tablename, sql_w)
    1130
    1131 -    def delete(self, tablename, query):
    1132          sql = self._delete(tablename, query)
    1133          ### special code to handle CASCADE in SQLite
    1134          db = self.db
    1135          table = db[tablename]
    1136          if self.dbengine=='sqlite' and table._referenced_by:
    1137              deleted = [x[table._id.name] for x in db(query).select(table._id)]
    1138          ### end special code to handle CASCADE in SQLite
    1139          self.execute(sql)
    1140          try:
    1141              counter = self.cursor.rowcount
    1142          except:
    1143              counter =  None
    1144          ### special code to handle CASCADE in SQLite
    1145          if self.dbengine=='sqlite' and counter:
    1146              for tablename,fieldname in table._referenced_by:
    1147                  f = db[tablename][fieldname]
    1148                  if f.type=='reference '+table._tablename and f.ondelete=='CASCADE':
    1149                      db(db[tablename][fieldname].belongs(deleted)).delete()
    1150          ### end special code to handle CASCADE in SQLite
    1151          return counter
    1152
    1153 -    def get_table(self, query):
    1154          tablenames = self.tables(query)
    1155          if len(tablenames)==1:
    1156              return tablenames[0]
    1157          elif len(tablenames)<1:
    1158              raise RuntimeError, "No table selected"
    1159          else:
    1160              raise RuntimeError, "Too many tables selected"
    1161
    1162 -    def expand_all(self, fields, tablenames):
    1163          new_fields = []
    1164          for item in fields:
    1165              if isinstance(item,SQLALL):
    1166                  new_fields += item.table
    1167              else:
    1168                  new_fields.append(item)
    1169          # ## if no fields specified take them all from the requested tables
    1170          if not new_fields:
    1171              for table in tablenames:
    1172                  for field in self.db[table]:
    1173                      new_fields.append(field)
    1174          return new_fields
    1175
    1176 -    def _select(self, query, fields, attributes):
    1177          for key in set(attributes.keys())-set(('orderby', 'groupby', 'limitby',
    1178                                                 'required', 'cache', 'left',
    1179                                                 'distinct', 'having', 'join',
    1180                                                 'for_update')):
    1181              raise SyntaxError, 'invalid select attribute: %s' % key
    1182
    1183          tablenames = self.tables(query)
    1184          for field in fields:
    1185              if isinstance(field, basestring) and regex_table_field.match(field):
    1186                  tn,fn = field.split('.')
    1187                  field = self.db[tn][fn]
    1188              for tablename in self.tables(field):
    1189                  if not tablename in tablenames:
    1190                      tablenames.append(tablename)
    1191
    1192          if use_common_filters(query):
    1193              query = self.common_filter(query,tablenames)
    1194
    1195          if len(tablenames) < 1:
    1196              raise SyntaxError, 'Set: no tables selected'
    1197          sql_f = ', '.join(map(self.expand, fields))
    1198          self._colnames = [c.strip() for c in sql_f.split(', ')]
    1199          if query:
    1200              sql_w = ' WHERE ' + self.expand(query)
    1201          else:
    1202              sql_w = ''
    1203          sql_o = ''
    1204          sql_s = ''
    1205          left = attributes.get('left', False)
    1206          inner_join = attributes.get('join', False)
    1207          distinct = attributes.get('distinct', False)
    1208          groupby = attributes.get('groupby', False)
    1209          orderby = attributes.get('orderby', False)
    1210          having = attributes.get('having', False)
    1211          limitby = attributes.get('limitby', False)
    1212          for_update = attributes.get('for_update', False)
    1213          if self.can_select_for_update is False and for_update is True:
    1214              raise SyntaxError, 'invalid select attribute: for_update'
    1215          if distinct is True:
    1216              sql_s += 'DISTINCT'
    1217          elif distinct:
    1218              sql_s += 'DISTINCT ON (%s)' % distinct
    1219          if inner_join:
    1220              icommand = self.JOIN()
    1221              if not isinstance(inner_join, (tuple, list)):
    1222                  inner_join = [inner_join]
    1223              ijoint = [t._tablename for t in inner_join if not isinstance(t,Expression)]
    1224              ijoinon = [t for t in inner_join if isinstance(t, Expression)]
    1225              itables_to_merge={} #issue 490
    1226              [itables_to_merge.update(dict.fromkeys(self.tables(t))) for t in ijoinon] # issue 490
    1227              ijoinont = [t.first._tablename for t in ijoinon]
    1228              [itables_to_merge.pop(t) for t in ijoinont if t in itables_to_merge] #issue 490
    1229              iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys() # issue 490
    1230              iexcluded = [t for t in tablenames if not t in iimportant_tablenames]
    1231          if left:
    1232              join = attributes['left']
    1233              command = self.LEFT_JOIN()
    1234              if not isinstance(join, (tuple, list)):
    1235                  join = [join]
    1236              joint = [t._tablename for t in join if not isinstance(t, Expression)]
    1237              joinon = [t for t in join if isinstance(t, Expression)]
    1238              #patch join+left patch (solves problem with ordering in left joins)
    1239              tables_to_merge={}
    1240              [tables_to_merge.update(dict.fromkeys(self.tables(t))) for t in joinon]
    1241              joinont = [t.first._tablename for t in joinon]
    1242              [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge]
    1243              important_tablenames = joint + joinont + tables_to_merge.keys()
    1244              excluded = [t for t in tablenames if not t in important_tablenames ]
    1245          def alias(t):
    1246              return str(self.db[t])
    1247          if inner_join and not left:
    1248              sql_t = ', '.join([alias(t) for t in iexcluded + itables_to_merge.keys()]) # issue 490
    1249              for t in ijoinon:
    1250                  sql_t += ' %s %s' % (icommand, str(t))
    1251          elif not inner_join and left:
    1252              sql_t = ', '.join([alias(t) for t in excluded + tables_to_merge.keys()])
    1253              if joint:
    1254                  sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
    1255              for t in joinon:
    1256                  sql_t += ' %s %s' % (command, str(t))
    1257          elif inner_join and left:
    1258              all_tables_in_query = set(important_tablenames + \
    1259                                        iimportant_tablenames + \
    1260                                        tablenames) # issue 490
    1261              tables_in_joinon = set(joinont + ijoinont) # issue 490
    1262              tables_not_in_joinon = all_tables_in_query.difference(tables_in_joinon) # issue 490
    1263              sql_t = ','.join([alias(t) for t in tables_not_in_joinon]) # issue 490
    1264              for t in ijoinon:
    1265                  sql_t += ' %s %s' % (icommand, str(t))
    1266              if joint:
    1267                  sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
    1268              for t in joinon:
    1269                  sql_t += ' %s %s' % (command, str(t))
    1270          else:
    1271              sql_t = ', '.join(alias(t) for t in tablenames)
    1272          if groupby:
    1273              if isinstance(groupby, (list, tuple)):
    1274                  groupby = xorify(groupby)
    1275              sql_o += ' GROUP BY %s' % self.expand(groupby)
    1276              if having:
    1277                  sql_o += ' HAVING %s' % attributes['having']
    1278          if orderby:
    1279              if isinstance(orderby, (list, tuple)):
    1280                  orderby = xorify(orderby)
    1281              if str(orderby) == '<random>':
    1282                  sql_o += ' ORDER BY %s' % self.RANDOM()
    1283              else:
    1284                  sql_o += ' ORDER BY %s' % self.expand(orderby)
    1285          if limitby:
    1286              if not orderby and tablenames:
    1287                  sql_o += ' ORDER BY %s' % ', '.join(['%s.%s'%(t,x) for t in tablenames for x in ((hasattr(self.db[t], '_primarykey') and self.db[t]._primarykey) or [self.db[t]._id.name])])
    1288              # oracle does not support limitby
    1289          sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby)
    1290          if for_update and self.can_select_for_update is True:
    1291              sql = sql.rstrip(';') + ' FOR UPDATE;'
    1292          return sql
    1293
    1294 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    1295          if limitby:
    1296              (lmin, lmax) = limitby
    1297              sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
    1298          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    1299
    1300 -    def select(self, query, fields, attributes):
    1301          """
    1302          Always returns a Rows object, possibly empty.
    1303          """
    1304          def response(sql):
    1305              self.execute(sql)
    1306              return self.cursor.fetchall()
    1307          sql = self._select(query, fields, attributes)
    1308          if attributes.get('cache', None):
    1309              (cache_model, time_expire) = attributes['cache']
    1310              del attributes['cache']
    1311              key = self.uri + '/' + sql
    1312              key = (len(key)<=200) and key or hashlib.md5(key).hexdigest()
    1313              rows = cache_model(key, lambda: response(sql), time_expire)
    1314          else:
    1315              rows = response(sql)
    1316          if isinstance(rows,tuple):
    1317              rows = list(rows)
    1318          limitby = attributes.get('limitby', None) or (0,)
    1319          rows = self.rowslice(rows,limitby[0],None)
    1320          return self.parse(rows,fields,self._colnames)
    1321
    1322 -    def _count(self, query, distinct=None):
    1323          tablenames = self.tables(query)
    1324          if query:
    1325              if use_common_filters(query):
    1326                  query = self.common_filter(query, tablenames)
    1327              sql_w = ' WHERE ' + self.expand(query)
    1328          else:
    1329              sql_w = ''
    1330          sql_t = ','.join(tablenames)
    1331          if distinct:
    1332              if isinstance(distinct,(list, tuple)):
    1333                  distinct = xorify(distinct)
    1334              sql_d = self.expand(distinct)
    1335              return 'SELECT count(DISTINCT %s) FROM %s%s;' % (sql_d, sql_t, sql_w)
    1336          return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
    1337
    1338 -    def count(self, query, distinct=None):
    1339          self.execute(self._count(query, distinct))
    1340          return self.cursor.fetchone()[0]
    1341
    1342 -    def tables(self, query):
    1343          tables = set()
    1344          if isinstance(query, Field):
    1345              tables.add(query.tablename)
    1346          elif isinstance(query, (Expression, Query)):
    1347              if not query.first is None:
    1348                  tables = tables.union(self.tables(query.first))
    1349              if not query.second is None:
    1350                  tables = tables.union(self.tables(query.second))
    1351          return list(tables)
    1352
    1353 -    def commit(self):
    1354          return self.connection.commit()
    1355
    1356 -    def rollback(self):
    1357          return self.connection.rollback()
    1358
    1359 -    def close(self):
    1360          return self.connection.close()
    1361
    1362 -    def distributed_transaction_begin(self, key):
    1363          return
    1364
    1365 -    def prepare(self, key):
    1366          self.connection.prepare()
    1367
    1368 -    def commit_prepared(self, key):
    1369          self.connection.commit()
    1370
    1371 -    def rollback_prepared(self, key):
    1372          self.connection.rollback()
    1373
    1374 -    def concat_add(self, table):
    1375          return ', ADD '
    1376
    1377 -    def constraint_name(self, table, fieldname):
    1378          return '%s_%s__constraint' % (table,fieldname)
    1379
    1380 -    def create_sequence_and_triggers(self, query, table, **args):
    1381          self.execute(query)
    1382
    1383 -    def log_execute(self, *a, **b):
    1384          self.db._lastsql = a[0]
    1385          t0 = time.time()
    1386          ret = self.cursor.execute(*a, **b)
    1387          self.db._timings.append((a[0],time.time()-t0))
    1388          del self.db._timings[:-TIMINGSSIZE]
    1389          return ret
    1390
    1391 -    def execute(self, *a, **b):
    1392          return self.log_execute(*a, **b)
    1393
    1394 -    def represent(self, obj, fieldtype):
    1395          if isinstance(obj, CALLABLETYPES):
    1396              obj = obj()
    1397          if isinstance(fieldtype, SQLCustomType):
    1398              return fieldtype.encoder(obj)
    1399          if isinstance(obj, (Expression, Field)):
    1400              return str(obj)
    1401          if fieldtype.startswith('list:'):
    1402              if not obj:
    1403                  obj = []
    1404              elif not isinstance(obj, (list, tuple)):
    1405                  obj = [int(obj)]
    1406              elif fieldtype.startswith('list:string'):
    1407                  obj = [str(item) for item in obj]
    1408              else:
    1409                  obj = [int(item) for item in obj]
    1410          if isinstance(obj, (list, tuple)):
    1411              obj = bar_encode(obj)
    1412          if obj is None:
    1413              return 'NULL'
    1414          if obj == '' and not fieldtype[:2] in ['st', 'te', 'pa', 'up']:
    1415              return 'NULL'
    1416          r = self.represent_exceptions(obj, fieldtype)
    1417          if not r is None:
    1418              return r
    1419          if fieldtype == 'boolean':
    1420              if obj and not str(obj)[:1].upper() in ['F', '0']:
    1421                  return "'T'"
    1422              else:
    1423                  return "'F'"
    1424          if fieldtype == 'id' or fieldtype == 'integer':
    1425              return str(int(obj))
    1426          if fieldtype.startswith('decimal'):
    1427              return str(obj)
    1428          elif fieldtype.startswith('reference'): # reference
    1429              if fieldtype.find('.')>0:
    1430                  return repr(obj)
    1431              elif isinstance(obj, (Row, Reference)):
    1432                  return str(obj['id'])
    1433              return str(int(obj))
    1434          elif fieldtype == 'double':
    1435              return repr(float(obj))
    1436          if isinstance(obj, unicode):
    1437              obj = obj.encode(self.db_codec)
    1438          if fieldtype == 'blob':
    1439              obj = base64.b64encode(str(obj))
    1440          elif fieldtype == 'date':
    1441              if isinstance(obj, (datetime.date, datetime.datetime)):
    1442                  obj = obj.isoformat()[:10]
    1443              else:
    1444                  obj = str(obj)
    1445          elif fieldtype == 'datetime':
    1446              if isinstance(obj, datetime.datetime):
    1447                  obj = obj.isoformat()[:19].replace('T',' ')
    1448              elif isinstance(obj, datetime.date):
    1449                  obj = obj.isoformat()[:10]+' 00:00:00'
    1450              else:
    1451                  obj = str(obj)
    1452          elif fieldtype == 'time':
    1453              if isinstance(obj, datetime.time):
    1454                  obj = obj.isoformat()[:10]
    1455              else:
    1456                  obj = str(obj)
    1457          if not isinstance(obj,str):
    1458              obj = str(obj)
    1459          try:
    1460              obj.decode(self.db_codec)
    1461          except:
    1462              obj = obj.decode('latin1').encode(self.db_codec)
    1463          return self.adapt(obj)
    1464
    1465 -    def represent_exceptions(self, obj, fieldtype):
    1466          return None
    1467
    1468 -    def lastrowid(self, table):
    1469          return None
    1470
    1471 -    def integrity_error_class(self):
    1472          return type(None)
    1473
    1474 -    def rowslice(self, rows, minimum=0, maximum=None):
    1475          """ By default this function does nothing; overload when db does not do slicing. """
    1476          return rows
    1477
    1478 -    def parse_value(self, value, field_type, blob_decode=True):
    1479          if field_type != 'blob' and isinstance(value, str):
    1480              try:
    1481                  value = value.decode(self.db._db_codec)
    1482              except Exception:
    1483                  pass
    1484          if isinstance(value, unicode):
    1485              value = value.encode('utf-8')
    1486          elif isinstance(field_type, SQLCustomType):
    1487              value = field_type.decoder(value)
    1488          if not isinstance(field_type, str) or value is None:
    1489              return value
    1490          elif field_type in ('string', 'text', 'password', 'upload'):
    1491              return value
    1492          elif field_type == 'blob' and not blob_decode:
    1493              return value
    1494          else:
    1495              key = regex_type.match(field_type).group(0)
    1496              return self.parsemap[key](value,field_type)
    1497
    1498 -    def parse_reference(self, value, field_type):
    1499          referee = field_type[10:].strip()
    1500          if not '.' in referee:
    1501              value = Reference(value)
    1502              value._table, value._record = self.db[referee], None
    1503          return value
    1504
    1505 -    def parse_boolean(self, value, field_type):
    1506          return value == True or str(value)[:1].lower() == 't'
    1507
    1508 -    def parse_date(self, value, field_type):
    1509          if not isinstance(value, (datetime.date,datetime.datetime)):
    1510              (y, m, d) = map(int, str(value)[:10].strip().split('-'))
    1511              value = datetime.date(y, m, d)
    1512          return value
    1513
    1514 -    def parse_time(self, value, field_type):
    1515          if not isinstance(value, datetime.time):
    1516              time_items = map(int,str(value)[:8].strip().split(':')[:3])
    1517              if len(time_items) == 3:
    1518                  (h, mi, s) = time_items
    1519              else:
    1520                  (h, mi, s) = time_items + [0]
    1521              value = datetime.time(h, mi, s)
    1522          return value
    1523
    1524 -    def parse_datetime(self, value, field_type):
    1525          if not isinstance(value, datetime.datetime):
    1526              (y, m, d) = map(int,str(value)[:10].strip().split('-'))
    1527              time_items = map(int,str(value)[11:19].strip().split(':')[:3])
    1528              if len(time_items) == 3:
    1529                  (h, mi, s) = time_items
    1530              else:
    1531                  (h, mi, s) = time_items + [0]
    1532              value = datetime.datetime(y, m, d, h, mi, s)
    1533          return value
    1534
    1535 -    def parse_blob(self, value, field_type):
    1536          return base64.b64decode(str(value))
    1537
    1538 -    def parse_decimal(self, value, field_type):
    1539          decimals = int(field_type[8:-1].split(',')[-1])
    1540          if self.dbengine == 'sqlite':
    1541              value = ('%.' + str(decimals) + 'f') % value
    1542          if not isinstance(value, decimal.Decimal):
    1543              value = decimal.Decimal(str(value))
    1544          return value
    1545
    1546 -    def parse_list_integers(self, value, field_type):
    1547          if not self.dbengine=='google:datastore':
    1548              value = bar_decode_integer(value)
    1549          return value
    1550
    1551 -    def parse_list_references(self, value, field_type):
    1552          if not self.dbengine=='google:datastore':
    1553              value = bar_decode_integer(value)
    1554          return [self.parse_reference(r, field_type[5:]) for r in value]
    1555
    1556 -    def parse_list_strings(self, value, field_type):
    1557          if not self.dbengine=='google:datastore':
    1558              value = bar_decode_string(value)
    1559          return value
    1560
    1561 -    def parse_id(self, value, field_type):
    1562          return int(value)
    1563
    1564 -    def parse_integer(self, value, field_type):
    1565          return int(value)
    1566
    1567 -    def parse_double(self, value, field_type):
    1568          return float(value)
    1569
    1570 -    def build_parsemap(self):
    1571          self.parsemap = {
    1572              'id':self.parse_id,
    1573              'integer':self.parse_integer,
    1574              'double':self.parse_double,
    1575              'reference':self.parse_reference,
    1576              'boolean':self.parse_boolean,
    1577              'date':self.parse_date,
    1578              'time':self.parse_time,
    1579              'datetime':self.parse_datetime,
    1580              'blob':self.parse_blob,
    1581              'decimal':self.parse_decimal,
    1582              'list:integer':self.parse_list_integers,
    1583              'list:reference':self.parse_list_references,
    1584              'list:string':self.parse_list_strings,
    1585              }
    1586
    1587 -    def parse(self, rows, fields, colnames, blob_decode=True):
    1588          self.build_parsemap()
    1589          db = self.db
    1590          virtualtables = []
    1591          new_rows = []
    1592          for (i,row) in enumerate(rows):
    1593              new_row = Row()
    1594              for j,colname in enumerate(colnames):
    1595                  value = row[j]
    1596                  if not regex_table_field.match(colnames[j]):
    1597                      if not '_extra' in new_row:
    1598                          new_row['_extra'] = Row()
    1599                      new_row['_extra'][colnames[j]] = \
    1600                          self.parse_value(value, fields[j].type,blob_decode)
    1601                      new_column_name = \
    1602                          regex_select_as_parser.search(colnames[j])
    1603                      if not new_column_name is None:
    1604                          column_name = new_column_name.groups(0)
    1605                          setattr(new_row,column_name[0],value)
    1606                  else:
    1607                      (tablename, fieldname) = colname.split('.')
    1608                      table = db[tablename]
    1609                      field = table[fieldname]
    1610                      if not tablename in new_row:
    1611                          colset = new_row[tablename] = Row()
    1612                          if tablename not in virtualtables:
    1613                              virtualtables.append(tablename)
    1614                      else:
    1615                          colset = new_row[tablename]
    1616                      colset[fieldname] = value = \
    1617                          self.parse_value(value,field.type,blob_decode)
    1618
    1619                      if field.type == 'id':
    1620                          id = value
    1621                          colset.update_record = lambda _ = (colset, table, id), **a: update_record(_, a)
    1622                          colset.delete_record = lambda t = table, i = id: t._db(t._id==i).delete()
    1623                          for (referee_table, referee_name) in table._referenced_by:
    1624                              s = db[referee_table][referee_name]
    1625                              referee_link = db._referee_name and \
    1626                                  db._referee_name % dict(table=referee_table,field=referee_name)
    1627                              if referee_link and not referee_link in colset:
    1628                                  colset[referee_link] = Set(db, s == id)
    1629              new_rows.append(new_row)
    1630          rowsobj = Rows(db, new_rows, colnames, rawrows=rows)
    1631
    1632          for tablename in virtualtables:
    1633              ### new style virtual fields
    1634              table = db[tablename]
    1635              fields_virtual = [(f,v) for (f,v) in table.items() if isinstance(v,FieldVirtual)]
    1636              fields_lazy = [(f,v) for (f,v) in table.items() if isinstance(v,FieldLazy)]
    1637              if fields_virtual or fields_lazy:
    1638                  for row in rowsobj.records:
    1639                      box = row[tablename]
    1640                      for f,v in fields_virtual:
    1641                          box[f] = v.f(row)
    1642                      for f,v in fields_lazy:
    1643                          box[f] = (v.handler or VirtualCommand)(v.f,row)
    1644
    1645              ### old style virtual fields
    1646              for item in table.virtualfields:
    1647                  try:
    1648                      rowsobj = rowsobj.setvirtualfields(**{tablename:item})
    1649                  except KeyError:
    1650                      # to avoid breaking virtualfields when partial select
    1651                      pass
    1652          return rowsobj
    1653
    1654 -    def common_filter(self, query, tablenames):
    1655          tenant_fieldname = self.db._request_tenant
    1656
    1657          for tablename in tablenames:
    1658              table = self.db[tablename]
    1659
    1660              # deal with user provided filters
    1661              if table._common_filter != None:
    1662                  query = query & table._common_filter(query)
    1663
    1664              # deal with multi_tenant filters
    1665              if tenant_fieldname in table:
    1666                  default = table[tenant_fieldname].default
    1667                  if not default is None:
    1668                      newquery = table[tenant_fieldname] == default
    1669                      if query is None:
    1670                          query = newquery
    1671                      else:
    1672                          query = query & newquery
    1673          return query
    1674
    1675  ###################################################################################
    1676  # List of all the available adapters; they all extend BaseAdapter.
    1677  ###################################################################################
    1678
    1679 -class SQLiteAdapter(BaseAdapter):
    1680
    1681      driver = globals().get('sqlite3', None)
    1682      can_select_for_update = None    # support ourselves with BEGIN TRANSACTION
    1683
    1684 -    def EXTRACT(self,field,what):
    1685          return "web2py_extract('%s',%s)" % (what, self.expand(field))
    1686
    1687      @staticmethod
    1688 -    def web2py_extract(lookup, s):
    1689          table = {
    1690              'year': (0, 4),
    1691              'month': (5, 7),
    1692              'day': (8, 10),
    1693              'hour': (11, 13),
    1694              'minute': (14, 16),
    1695              'second': (17, 19),
    1696              }
    1697          try:
    1698              (i, j) = table[lookup]
    1699              return int(s[i:j])
    1700          except:
    1701              return None
    1702
    1703      @staticmethod
    1704 -    def web2py_regexp(expression, item):
    1705          return re.compile(expression).search(item) is not None
    1706
    1707 -    def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
    1708                   credential_decoder=lambda x:x, driver_args={},
    1709                   adapter_args={}):
    1710          if not self.driver:
    1711              raise RuntimeError, "Unable to import driver"
    1712          self.db = db
    1713          self.dbengine = "sqlite"
    1714          self.uri = uri
    1715          self.pool_size = 0
    1716          self.folder = folder
    1717          self.db_codec = db_codec
    1718          self.find_or_make_work_folder()
    1719          path_encoding = sys.getfilesystemencoding() or locale.getdefaultlocale()[1] or 'utf8'
    1720          if uri.startswith('sqlite:memory'):
    1721              dbpath = ':memory:'
    1722          else:
    1723              dbpath = uri.split('://')[1]
    1724              if dbpath[0] != '/':
    1725                  dbpath = os.path.join(self.folder.decode(path_encoding).encode('utf8'), dbpath)
    1726          if not 'check_same_thread' in driver_args:
    1727              driver_args['check_same_thread'] = False
    1728          if not 'detect_types' in driver_args:
    1729              driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
    1730          def connect(dbpath=dbpath, driver_args=driver_args):
    1731              return self.driver.Connection(dbpath, **driver_args)
    1732          self.pool_connection(connect)
    1733          self.connection.create_function('web2py_extract', 2,
    1734                                          SQLiteAdapter.web2py_extract)
    1735          self.connection.create_function("REGEXP", 2,
    1736                                          SQLiteAdapter.web2py_regexp)
    1737
    1738 -    def _truncate(self, table, mode=''):
    1739          tablename = table._tablename
    1740          return ['DELETE FROM %s;' % tablename,
    1741                  "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
    1742
    1743 -    def lastrowid(self, table):
    1744          return self.cursor.lastrowid
    1745
    1746 -    def REGEXP(self,first,second):
    1747          return '(%s REGEXP %s)' % (self.expand(first),
    1748                                     self.expand(second,'string'))
    1749
    1750 -    def _select(self, query, fields, attributes):
    1751          """
    1752          Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION.
    1753          Note that the entire database, rather than one record, is locked
    1754          (it will be locked eventually anyway by the following UPDATE).
    1755          """
    1756          sql = super(SQLiteAdapter, self)._select(query, fields, attributes)
    1757          if attributes.get('for_update', False):
    1758              sql = 'BEGIN IMMEDIATE TRANSACTION; ' + sql
    1759          return sql
    1760
    1761
    1762 -class JDBCSQLiteAdapter(SQLiteAdapter):
    1763
    1764      driver = globals().get('zxJDBC', None)
    1765
    1766 -    def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
    1767                   credential_decoder=lambda x:x, driver_args={},
    1768                   adapter_args={}):
    1769          if not self.driver:
    1770              raise RuntimeError, "Unable to import driver"
    1771          self.db = db
    1772          self.dbengine = "sqlite"
    1773          self.uri = uri
    1774          self.pool_size = pool_size
    1775          self.folder = folder
    1776          self.db_codec = db_codec
    1777          self.find_or_make_work_folder()
    1778          path_encoding = sys.getfilesystemencoding() or locale.getdefaultlocale()[1] or 'utf8'
    1779          if uri.startswith('sqlite:memory'):
    1780              dbpath = ':memory:'
    1781          else:
    1782              dbpath = uri.split('://')[1]
    1783              if dbpath[0] != '/':
    1784                  dbpath = os.path.join(self.folder.decode(path_encoding).encode('utf8'), dbpath)
    1785          def connect(dbpath=dbpath,driver_args=driver_args):
    1786              return self.driver.connect(java.sql.DriverManager.getConnection('jdbc:sqlite:'+dbpath), **driver_args)
    1787          self.pool_connection(connect)
    1788          # FIXME http://www.zentus.com/sqlitejdbc/custom_functions.html for UDFs
    1789          self.connection.create_function('web2py_extract', 2, SQLiteAdapter.web2py_extract)
    1790
    1791 -    def execute(self, a):
    1792          return self.log_execute(a)
    1793
    1794
    1795 -class MySQLAdapter(BaseAdapter):
    1796
    1797      driver = globals().get('pymysql',None)
    1798      maxcharlength = 255
    1799      commit_on_alter_table = True
    1800      support_distributed_transaction = True
    1801      types = {
    1802          'boolean': 'CHAR(1)',
    1803          'string': 'VARCHAR(%(length)s)',
    1804          'text': 'LONGTEXT',
    1805          'password': 'VARCHAR(%(length)s)',
    1806          'blob': 'LONGBLOB',
    1807          'upload': 'VARCHAR(%(length)s)',
    1808          'integer': 'INT',
    1809          'double': 'DOUBLE',
    1810          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    1811          'date': 'DATE',
    1812          'time': 'TIME',
    1813          'datetime': 'DATETIME',
    1814          'id': 'INT AUTO_INCREMENT NOT NULL',
    1815          'reference': 'INT, INDEX %(field_name)s__idx (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    1816          'list:integer': 'LONGTEXT',
    1817          'list:string': 'LONGTEXT',
    1818          'list:reference': 'LONGTEXT',
    1819          }
    1820
    1821 -    def RANDOM(self):
    1822          return 'RAND()'
    1823
    1824 -    def SUBSTRING(self,field,parameters):
    1825          return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
    1826
    1827 -    def _drop(self,table,mode):
    1828          # breaks db integrity but without this mysql does not drop table
    1829          return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table,'SET FOREIGN_KEY_CHECKS=1;']
    1830
    1831 -    def distributed_transaction_begin(self,key):
    1832          self.execute('XA START;')
    1833
    1834 -    def prepare(self,key):
    1835          self.execute("XA END;")
    1836          self.execute("XA PREPARE;")
    1837
    1838 -    def commit_prepared(self,ley):
    1839          self.execute("XA COMMIT;")
    1840
    1841 -    def rollback_prepared(self,key):
    1842          self.execute("XA ROLLBACK;")
    1843
    1844 -    def concat_add(self,table):
    1845          return '; ALTER TABLE %s ADD ' % table
    1846
    1847 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    1848                   credential_decoder=lambda x:x, driver_args={},
    1849                   adapter_args={}):
    1850          if not self.driver:
    1851              raise RuntimeError, "Unable to import driver"
    1852          self.db = db
    1853          self.dbengine = "mysql"
    1854          self.uri = uri
    1855          self.pool_size = pool_size
    1856          self.folder = folder
    1857          self.db_codec = db_codec
    1858          self.find_or_make_work_folder()
    1859          uri = uri.split('://')[1]
    1860          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$').match(uri)
    1861          if not m:
    1862              raise SyntaxError, \
    1863                  "Invalid URI string in DAL: %s" % self.uri
    1864          user = credential_decoder(m.group('user'))
    1865          if not user:
    1866              raise SyntaxError, 'User required'
    1867          password = credential_decoder(m.group('password'))
    1868          if not password:
    1869              password = ''
    1870          host = m.group('host')
    1871          if not host:
    1872              raise SyntaxError, 'Host name required'
    1873          db = m.group('db')
    1874          if not db:
    1875              raise SyntaxError, 'Database name required'
    1876          port = int(m.group('port') or '3306')
    1877          charset = m.group('charset') or 'utf8'
    1878          driver_args.update(dict(db=db,
    1879                                  user=credential_decoder(user),
    1880                                  passwd=credential_decoder(password),
    1881                                  host=host,
    1882                                  port=port,
    1883                                  charset=charset))
    1884          def connect(driver_args=driver_args):
    1885              return self.driver.connect(**driver_args)
    1886          self.pool_connection(connect)
    1887          self.execute('SET FOREIGN_KEY_CHECKS=1;')
    1888          self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
    1889
    1890 -    def lastrowid(self,table):
    1891          self.execute('select last_insert_id();')
    1892          return int(self.cursor.fetchone()[0])
    1893
    1894 -class PostgreSQLAdapter(BaseAdapter):
    1895
    1896      driver = None
    1897      drivers = {'psycopg2': globals().get('psycopg2', None),
    1898                 'pg8000': globals().get('pg8000', None), }
    1899
    1900      support_distributed_transaction = True
    1901      types = {
    1902          'boolean': 'CHAR(1)',
    1903          'string': 'VARCHAR(%(length)s)',
    1904          'text': 'TEXT',
    1905          'password': 'VARCHAR(%(length)s)',
    1906          'blob': 'BYTEA',
    1907          'upload': 'VARCHAR(%(length)s)',
    1908          'integer': 'INTEGER',
    1909          'double': 'FLOAT8',
    1910          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    1911          'date': 'DATE',
    1912          'time': 'TIME',
    1913          'datetime': 'TIMESTAMP',
    1914          'id': 'SERIAL PRIMARY KEY',
    1915          'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    1916          'list:integer': 'TEXT',
    1917          'list:string': 'TEXT',
    1918          'list:reference': 'TEXT',
    1919          }
    1920
    1921 -    def adapt(self,obj):
    1922          return psycopg2_adapt(obj).getquoted()
    1923
    1924 -    def sequence_name(self,table):
    1925          return '%s_id_Seq' % table
    1926
    1927 -    def RANDOM(self):
    1928          return 'RANDOM()'
    1929
    1930 -    def ADD(self, first, second):
    1931          t = first.type
    1932          if t in ('text','string','password','upload','blob'):
    1933              return '(%s || %s)' % (self.expand(first), self.expand(second, t))
    1934          else:
    1935              return '(%s + %s)' % (self.expand(first), self.expand(second, t))
    1936
    1937 -    def distributed_transaction_begin(self,key):
    1938          return
    1939
    1940 -    def prepare(self,key):
    1941          self.execute("PREPARE TRANSACTION '%s';" % key)
    1942
    1943 -    def commit_prepared(self,key):
    1944          self.execute("COMMIT PREPARED '%s';" % key)
    1945
    1946 -    def rollback_prepared(self,key):
    1947          self.execute("ROLLBACK PREPARED '%s';" % key)
    1948
    1949 -    def create_sequence_and_triggers(self, query, table, **args):
    1950          # following lines should only be executed if table._sequence_name does not exist
    1951          # self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
    1952          # self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
    1953          #              % (table._tablename, table._fieldname, table._sequence_name))
    1954          self.execute(query)
    1955
    1956 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    1957                   credential_decoder=lambda x:x, driver_args={},
    1958                   adapter_args={}):
    1959          if not self.drivers.get('psycopg2') and not self.drivers.get('pg8000'):
    1960              raise RuntimeError, "Unable to import any drivers (psycopg2 or pg8000)"
    1961          self.db = db
    1962          self.dbengine = "postgres"
    1963          self.uri = uri
    1964          self.pool_size = pool_size
    1965          self.folder = folder
    1966          self.db_codec = db_codec
    1967          self.find_or_make_work_folder()
    1968          library, uri = uri.split('://')[:2]
    1969          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$').match(uri)
    1970          if not m:
    1971              raise SyntaxError, "Invalid URI string in DAL"
    1972          user = credential_decoder(m.group('user'))
    1973          if not user:
    1974              raise SyntaxError, 'User required'
    1975          password = credential_decoder(m.group('password'))
    1976          if not password:
    1977              password = ''
    1978          host = m.group('host')
    1979          if not host:
    1980              raise SyntaxError, 'Host name required'
    1981          db = m.group('db')
    1982          if not db:
    1983              raise SyntaxError, 'Database name required'
    1984          port = m.group('port') or '5432'
    1985          sslmode = m.group('sslmode')
    1986          if sslmode:
    1987              msg = ("dbname='%s' user='%s' host='%s' "
    1988                     "port=%s password='%s' sslmode='%s'") \
    1989                     % (db, user, host, port, password, sslmode)
    1990          else:
    1991              msg = ("dbname='%s' user='%s' host='%s' "
    1992                     "port=%s password='%s'") \
    1993                     % (db, user, host, port, password)
    1994          # choose diver according uri
    1995          if library == "postgres":
    1996              if self.drivers.get('psycopg2'):
    1997                  self.driver = self.drivers['psycopg2']
    1998              elif self.drivers.get('pg8000'):
    1999                  self.driver = drivers.get('pg8000')
    2000          elif library == "postgres:psycopg2":
    2001              self.driver = self.drivers.get('psycopg2')
    2002          elif library == "postgres:pg8000":
    2003              self.driver = self.drivers.get('pg8000')
    2004          if not self.driver:
    2005              raise RuntimeError, "%s is not available" % library
    2006
    2007          self.__version__ = "%s %s" % (self.driver.__name__, self.driver.__version__)
    2008          def connect(msg=msg,driver_args=driver_args):
    2009              return self.driver.connect(msg,**driver_args)
    2010          self.pool_connection(connect)
    2011          self.connection.set_client_encoding('UTF8')
    2012          self.execute("SET standard_conforming_strings=on;")
    2013
    2014 -    def lastrowid(self,table):
    2015          self.execute("select currval('%s')" % table._sequence_name)
    2016          return int(self.cursor.fetchone()[0])
    2017
    2018
    2019 -    def LIKE(self,first,second):
    2020          return '(%s LIKE %s)' % (self.expand(first),
    2021                                   self.expand(second,'string'))
    2022
    2023 -    def ILIKE(self,first,second):
    2024          return '(%s ILIKE %s)' % (self.expand(first),
    2025                                    self.expand(second,'string'))
    2026
    2027 -    def REGEXP(self,first,second):
    2028          return '(%s ~ %s)' % (self.expand(first),
    2029                                self.expand(second,'string'))
    2030
    2031 -    def STARTSWITH(self,first,second):
    2032          return '(%s ILIKE %s)' % (self.expand(first),
    2033                                    self.expand(second+'%','string'))
    2034
    2035 -    def ENDSWITH(self,first,second):
    2036          return '(%s ILIKE %s)' % (self.expand(first),
    2037                                    self.expand('%'+second,'string'))
    2038
    2039 -    def CONTAINS(self,first,second):
    2040          if first.type in ('string','text'):
    2041              key = '%'+str(second).replace('%','%%')+'%'
    2042          elif first.type.startswith('list:'):
    2043              key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%'
    2044          return '(%s ILIKE %s)' % (self.expand(first),self.expand(key,'string'))
    2045
    2046 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
    2047
    2048 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2049                   credential_decoder=lambda x:x, driver_args={},
    2050                   adapter_args={}):
    2051          if not self.driver:
    2052              raise RuntimeError, "Unable to import driver"
    2053          self.db = db
    2054          self.dbengine = "postgres"
    2055          self.uri = uri
    2056          self.pool_size = pool_size
    2057          self.folder = folder
    2058          self.db_codec = db_codec
    2059          self.find_or_make_work_folder()
    2060          uri = uri.split('://')[1]
    2061          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$').match(uri)
    2062          if not m:
    2063              raise SyntaxError, "Invalid URI string in DAL"
    2064          user = credential_decoder(m.group('user'))
    2065          if not user:
    2066              raise SyntaxError, 'User required'
    2067          password = credential_decoder(m.group('password'))
    2068          if not password:
    2069              password = ''
    2070          host = m.group('host')
    2071          if not host:
    2072              raise SyntaxError, 'Host name required'
    2073          db = m.group('db')
    2074          if not db:
    2075              raise SyntaxError, 'Database name required'
    2076          port = m.group('port') or '5432'
    2077          msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
    2078          def connect(msg=msg,driver_args=driver_args):
    2079              return self.driver.connect(*msg,**driver_args)
    2080          self.pool_connection(connect)
    2081          self.connection.set_client_encoding('UTF8')
    2082          self.execute('BEGIN;')
    2083          self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
    2084
    2085
    2086 -class OracleAdapter(BaseAdapter):
    2087
    2088      driver = globals().get('cx_Oracle',None)
    2089
    2090      commit_on_alter_table = False
    2091      types = {
    2092          'boolean': 'CHAR(1)',
    2093          'string': 'VARCHAR2(%(length)s)',
    2094          'text': 'CLOB',
    2095          'password': 'VARCHAR2(%(length)s)',
    2096          'blob': 'CLOB',
    2097          'upload': 'VARCHAR2(%(length)s)',
    2098          'integer': 'INT',
    2099          'double': 'FLOAT',
    2100          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    2101          'date': 'DATE',
    2102          'time': 'CHAR(8)',
    2103          'datetime': 'DATE',
    2104          'id': 'NUMBER PRIMARY KEY',
    2105          'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2106          'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2107          'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
    2108          'list:integer': 'CLOB',
    2109          'list:string': 'CLOB',
    2110          'list:reference': 'CLOB',
    2111          }
    2112
    2113 -    def sequence_name(self,tablename):
    2114          return '%s_sequence' % tablename
    2115
    2116 -    def trigger_name(self,tablename):
    2117          return '%s_trigger' % tablename
    2118
    2119 -    def LEFT_JOIN(self):
    2120          return 'LEFT OUTER JOIN'
    2121
    2122 -    def RANDOM(self):
    2123          return 'dbms_random.value'
    2124
    2125 -    def NOT_NULL(self,default,field_type):
    2126          return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
    2127
    2128 -    def _drop(self,table,mode):
    2129          sequence_name = table._sequence_name
    2130          return ['DROP TABLE %s %s;' % (table, mode), 'DROP SEQUENCE %s;' % sequence_name]
    2131
    2132 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    2133          if limitby:
    2134              (lmin, lmax) = limitby
    2135              if len(sql_w) > 1:
    2136                  sql_w_row = sql_w + ' AND w_row > %i' % lmin
    2137              else:
    2138                  sql_w_row = 'WHERE w_row > %i' % lmin
    2139              return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
    2140          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    2141
    2142 -    def constraint_name(self, tablename, fieldname):
    2143          constraint_name = BaseAdapter.constraint_name(self, tablename, fieldname)
    2144          if len(constraint_name)>30:
    2145              constraint_name = '%s_%s__constraint' % (tablename[:10], fieldname[:7])
    2146          return constraint_name
    2147
    2148 -    def represent_exceptions(self, obj, fieldtype):
    2149          if fieldtype == 'blob':
    2150              obj = base64.b64encode(str(obj))
    2151              return ":CLOB('%s')" % obj
    2152          elif fieldtype == 'date':
    2153              if isinstance(obj, (datetime.date, datetime.datetime)):
    2154                  obj = obj.isoformat()[:10]
    2155              else:
    2156                  obj = str(obj)
    2157              return "to_date('%s','yyyy-mm-dd')" % obj
    2158          elif fieldtype == 'datetime':
    2159              if isinstance(obj, datetime.datetime):
    2160                  obj = obj.isoformat()[:19].replace('T',' ')
    2161              elif isinstance(obj, datetime.date):
    2162                  obj = obj.isoformat()[:10]+' 00:00:00'
    2163              else:
    2164                  obj = str(obj)
    2165              return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
    2166          return None
    2167
    2168 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2169                   credential_decoder=lambda x:x, driver_args={},
    2170                   adapter_args={}):
    2171          if not self.driver:
    2172              raise RuntimeError, "Unable to import driver"
    2173          self.db = db
    2174          self.dbengine = "oracle"
    2175          self.uri = uri
    2176          self.pool_size = pool_size
    2177          self.folder = folder
    2178          self.db_codec = db_codec
    2179          self.find_or_make_work_folder()
    2180          uri = uri.split('://')[1]
    2181          if not 'threaded' in driver_args:
    2182              driver_args['threaded']=True
    2183          def connect(uri=uri,driver_args=driver_args):
    2184              return self.driver.connect(uri,**driver_args)
    2185          self.pool_connection(connect)
    2186          self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
    2187          self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
    2188      oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
    2189
    2190 -    def execute(self, command, args=None):
    2191          args = args or []
    2192          i = 1
    2193          while True:
    2194              m = self.oracle_fix.match(command)
    2195              if not m:
    2196                  break
    2197              command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
    2198              args.append(m.group('clob')[6:-2].replace("''", "'"))
    2199              i += 1
    2200          if command[-1:]==';':
    2201              command = command[:-1]
    2202          return self.log_execute(command, args)
    2203
    2204 -    def create_sequence_and_triggers(self, query, table, **args):
    2205          tablename = table._tablename
    2206          sequence_name = table._sequence_name
    2207          trigger_name = table._trigger_name
    2208          self.execute(query)
    2209          self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE;' % sequence_name)
    2210          self.execute('CREATE OR REPLACE TRIGGER %s BEFORE INSERT ON %s FOR EACH ROW BEGIN SELECT %s.nextval INTO :NEW.id FROM DUAL; END;\n' % (trigger_name, tablename, sequence_name))
    2211
    2212 -    def lastrowid(self,table):
    2213          sequence_name = table._sequence_name
    2214          self.execute('SELECT %s.currval FROM dual;' % sequence_name)
    2215          return int(self.cursor.fetchone()[0])
    2216
    2217
    2218 -class MSSQLAdapter(BaseAdapter):
    2219
    2220      driver = globals().get('pyodbc',None)
    2221
    2222      types = {
    2223          'boolean': 'BIT',
    2224          'string': 'VARCHAR(%(length)s)',
    2225          'text': 'TEXT',
    2226          'password': 'VARCHAR(%(length)s)',
    2227          'blob': 'IMAGE',
    2228          'upload': 'VARCHAR(%(length)s)',
    2229          'integer': 'INT',
    2230          'double': 'FLOAT',
    2231          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    2232          'date': 'DATETIME',
    2233          'time': 'CHAR(8)',
    2234          'datetime': 'DATETIME',
    2235          'id': 'INT IDENTITY PRIMARY KEY',
    2236          'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2237          'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2238          'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
    2239          'list:integer': 'TEXT',
    2240          'list:string': 'TEXT',
    2241          'list:reference': 'TEXT',
    2242          }
    2243
    2244 -    def EXTRACT(self,field,what):
    2245          return "DATEPART(%s,%s)" % (what, self.expand(field))
    2246
    2247 -    def LEFT_JOIN(self):
    2248          return 'LEFT OUTER JOIN'
    2249
    2250 -    def RANDOM(self):
    2251          return 'NEWID()'
    2252
    2253 -    def ALLOW_NULL(self):
    2254          return ' NULL'
    2255
    2256 -    def SUBSTRING(self,field,parameters):
    2257          return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
    2258
    2259 -    def PRIMARY_KEY(self,key):
    2260          return 'PRIMARY KEY CLUSTERED (%s)' % key
    2261
    2262 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    2263          if limitby:
    2264              (lmin, lmax) = limitby
    2265              sql_s += ' TOP %i' % lmax
    2266          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    2267
    2268 -    def represent_exceptions(self, obj, fieldtype):
    2269          if fieldtype == 'boolean':
    2270              if obj and not str(obj)[0].upper() == 'F':
    2271                  return '1'
    2272              else:
    2273                  return '0'
    2274          return None
    2275
    2276 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2277                   credential_decoder=lambda x:x, driver_args={},
    2278                      adapter_args={}, fake_connect=False):
    2279          if not self.driver:
    2280              raise RuntimeError, "Unable to import driver"
    2281          self.db = db
    2282          self.dbengine = "mssql"
    2283          self.uri = uri
    2284          self.pool_size = pool_size
    2285          self.folder = folder
    2286          self.db_codec = db_codec
    2287          self.find_or_make_work_folder()
    2288          # ## read: http://bytes.com/groups/python/460325-cx_oracle-utf8
    2289          uri = uri.split('://')[1]
    2290          if '@' not in uri:
    2291              try:
    2292                  m = re.compile('^(?P<dsn>.+)$').match(uri)
    2293                  if not m:
    2294                      raise SyntaxError, \
    2295                          'Parsing uri string(%s) has no result' % self.uri
    2296                  dsn = m.group('dsn')
    2297                  if not dsn:
    2298                      raise SyntaxError, 'DSN required'
    2299              except SyntaxError, e:
    2300                  logger.error('NdGpatch error')
    2301                  raise e
    2302              # was cnxn = 'DSN=%s' % dsn
    2303              cnxn = dsn
    2304          else:
    2305              m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$').match(uri)
    2306              if not m:
    2307                  raise SyntaxError, \
    2308                      "Invalid URI string in DAL: %s" % uri
    2309              user = credential_decoder(m.group('user'))
    2310              if not user:
    2311                  raise SyntaxError, 'User required'
    2312              password = credential_decoder(m.group('password'))
    2313              if not password:
    2314                  password = ''
    2315              host = m.group('host')
    2316              if not host:
    2317                  raise SyntaxError, 'Host name required'
    2318              db = m.group('db')
    2319              if not db:
    2320                  raise SyntaxError, 'Database name required'
    2321              port = m.group('port') or '1433'
    2322              # Parse the optional url name-value arg pairs after the '?'
    2323              # (in the form of arg1=value1&arg2=value2&...)
    2324              # Default values (drivers like FreeTDS insist on uppercase parameter keys)
    2325              argsdict = { 'DRIVER':'{SQL Server}' }
    2326              urlargs = m.group('urlargs') or ''
    2327              argpattern = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
    2328              for argmatch in argpattern.finditer(urlargs):
    2329                  argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
    2330              urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.items()])
    2331              cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
    2332                  % (host, port, db, user, password, urlargs)
    2333          def connect(cnxn=cnxn,driver_args=driver_args):
    2334              return self.driver.connect(cnxn,**driver_args)
    2335          if not fake_connect:
    2336              self.pool_connection(connect)
    2337
    2338 -    def lastrowid(self,table):
    2339          #self.execute('SELECT @@IDENTITY;')
    2340          self.execute('SELECT SCOPE_IDENTITY();')
    2341          return int(self.cursor.fetchone()[0])
    2342
    2343 -    def integrity_error_class(self):
    2344          return pyodbc.IntegrityError
    2345
    2346 -    def rowslice(self,rows,minimum=0,maximum=None):
    2347          if maximum is None:
    2348              return rows[minimum:]
    2349          return rows[minimum:maximum]
    2350
    2351
    2352 -class MSSQL2Adapter(MSSQLAdapter):
    2353      types = {
    2354          'boolean': 'CHAR(1)',
    2355          'string': 'NVARCHAR(%(length)s)',
    2356          'text': 'NTEXT',
    2357          'password': 'NVARCHAR(%(length)s)',
    2358          'blob': 'IMAGE',
    2359          'upload': 'NVARCHAR(%(length)s)',
    2360          'integer': 'INT',
    2361          'double': 'FLOAT',
    2362          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    2363          'date': 'DATETIME',
    2364          'time': 'CHAR(8)',
    2365          'datetime': 'DATETIME',
    2366          'id': 'INT IDENTITY PRIMARY KEY',
    2367          'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2368          'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2369          'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
    2370          'list:integer': 'NTEXT',
    2371          'list:string': 'NTEXT',
    2372          'list:reference': 'NTEXT',
    2373          }
    2374
    2375 -    def represent(self, obj, fieldtype):
    2376          value = BaseAdapter.represent(self, obj, fieldtype)
    2377          if (fieldtype == 'string' or fieldtype == 'text') and value[:1]=="'":
    2378              value = 'N'+value
    2379          return value
    2380
    2381 -    def execute(self,a):
    2382          return self.log_execute(a.decode('utf8'))
    2383
    2384
    2385 -class FireBirdAdapter(BaseAdapter):
    2386
    2387      driver = globals().get('pyodbc',None)
    2388
    2389      commit_on_alter_table = False
    2390      support_distributed_transaction = True
    2391      types = {
    2392          'boolean': 'CHAR(1)',
    2393          'string': 'VARCHAR(%(length)s)',
    2394          'text': 'BLOB SUB_TYPE 1',
    2395          'password': 'VARCHAR(%(length)s)',
    2396          'blob': 'BLOB SUB_TYPE 0',
    2397          'upload': 'VARCHAR(%(length)s)',
    2398          'integer': 'INTEGER',
    2399          'double': 'DOUBLE PRECISION',
    2400          'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
    2401          'date': 'DATE',
    2402          'time': 'TIME',
    2403          'datetime': 'TIMESTAMP',
    2404          'id': 'INTEGER PRIMARY KEY',
    2405          'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2406          'list:integer': 'BLOB SUB_TYPE 1',
    2407          'list:string': 'BLOB SUB_TYPE 1',
    2408          'list:reference': 'BLOB SUB_TYPE 1',
    2409          }
    2410
    2411 -    def sequence_name(self,tablename):
    2412          return 'genid_%s' % tablename
    2413
    2414 -    def trigger_name(self,tablename):
    2415          return 'trg_id_%s' % tablename
    2416
    2417 -    def RANDOM(self):
    2418          return 'RAND()'
    2419
    2420 -    def NOT_NULL(self,default,field_type):
    2421          return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
    2422
    2423 -    def SUBSTRING(self,field,parameters):
    2424          return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
    2425
    2426 -    def _drop(self,table,mode):
    2427          sequence_name = table._sequence_name
    2428          return ['DROP TABLE %s %s;' % (table, mode), 'DROP GENERATOR %s;' % sequence_name]
    2429
    2430 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    2431          if limitby:
    2432              (lmin, lmax) = limitby
    2433              sql_s += ' FIRST %i SKIP %i' % (lmax - lmin, lmin)
    2434          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    2435
    2436 -    def _truncate(self,table,mode = ''):
    2437          return ['DELETE FROM %s;' % table._tablename,
    2438                  'SET GENERATOR %s TO 0;' % table._sequence_name]
    2439
    2440 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2441                   credential_decoder=lambda x:x, driver_args={},
    2442                   adapter_args={}):
    2443          if adapter_args.has_key('driver_name'):
    2444              if adapter_args['driver_name'] == 'kinterbasdb':
    2445                  self.driver = kinterbasdb
    2446              elif adapter_args['driver_name'] == 'firebirdsql':
    2447                  self.driver = firebirdsql
    2448          else:
    2449              self.driver = kinterbasdb
    2450
    2451          if not self.driver:
    2452              raise RuntimeError, "Unable to import driver"
    2453          self.db = db
    2454          self.dbengine = "firebird"
    2455          self.uri = uri
    2456          self.pool_size = pool_size
    2457          self.folder = folder
    2458          self.db_codec = db_codec
    2459          self.find_or_make_work_folder()
    2460          uri = uri.split('://')[1]
    2461          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$').match(uri)
    2462          if not m:
    2463              raise SyntaxError, "Invalid URI string in DAL: %s" % uri
    2464          user = credential_decoder(m.group('user'))
    2465          if not user:
    2466              raise SyntaxError, 'User required'
    2467          password = credential_decoder(m.group('password'))
    2468          if not password:
    2469              password = ''
    2470          host = m.group('host')
    2471          if not host:
    2472              raise SyntaxError, 'Host name required'
    2473          port = int(m.group('port') or 3050)
    2474          db = m.group('db')
    2475          if not db:
    2476              raise SyntaxError, 'Database name required'
    2477          charset = m.group('charset') or 'UTF8'
    2478          driver_args.update(dict(dsn='%s/%s:%s' % (host,port,db),
    2479                                     user = credential_decoder(user),
    2480                                     password = credential_decoder(password),
    2481                                     charset = charset))
    2482
    2483          def connect(driver_args=driver_args):
    2484              return self.driver.connect(**driver_args)
    2485          self.pool_connection(connect)
    2486
    2487 -    def create_sequence_and_triggers(self, query, table, **args):
    2488          tablename = table._tablename
    2489          sequence_name = table._sequence_name
    2490          trigger_name = table._trigger_name
    2491          self.execute(query)
    2492          self.execute('create generator %s;' % sequence_name)
    2493          self.execute('set generator %s to 0;' % sequence_name)
    2494          self.execute('create trigger %s for %s active before insert position 0 as\nbegin\nif(new.id is null) then\nbegin\nnew.id = gen_id(%s, 1);\nend\nend;' % (trigger_name, tablename, sequence_name))
    2495
    2496 -    def lastrowid(self,table):
    2497          sequence_name = table._sequence_name
    2498          self.execute('SELECT gen_id(%s, 0) FROM rdb$database' % sequence_name)
    2499          return int(self.cursor.fetchone()[0])
    2500
    2501
    2502 -class FireBirdEmbeddedAdapter(FireBirdAdapter):
    2503
    2504 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2505                   credential_decoder=lambda x:x, driver_args={},
    2506                   adapter_args={}):
    2507
    2508          if adapter_args.has_key('driver_name'):
    2509              if adapter_args['driver_name'] == 'kinterbasdb':
    2510                  self.driver = kinterbasdb
    2511              elif adapter_args['driver_name'] == 'firebirdsql':
    2512                  self.driver = firebirdsql
    2513          else:
    2514              self.driver = kinterbasdb
    2515
    2516          if not self.driver:
    2517              raise RuntimeError, "Unable to import driver"
    2518          self.db = db
    2519          self.dbengine = "firebird"
    2520          self.uri = uri
    2521          self.pool_size = pool_size
    2522          self.folder = folder
    2523          self.db_codec = db_codec
    2524          self.find_or_make_work_folder()
    2525          uri = uri.split('://')[1]
    2526          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$').match(uri)
    2527          if not m:
    2528              raise SyntaxError, \
    2529                  "Invalid URI string in DAL: %s" % self.uri
    2530          user = credential_decoder(m.group('user'))
    2531          if not user:
    2532              raise SyntaxError, 'User required'
    2533          password = credential_decoder(m.group('password'))
    2534          if not password:
    2535              password = ''
    2536          pathdb = m.group('path')
    2537          if not pathdb:
    2538              raise SyntaxError, 'Path required'
    2539          charset = m.group('charset')
    2540          if not charset:
    2541              charset = 'UTF8'
    2542          host = ''
    2543          driver_args.update(dict(host=host,
    2544                                     database=pathdb,
    2545                                     user=credential_decoder(user),
    2546                                     password=credential_decoder(password),
    2547                                     charset=charset))
    2548          #def connect(driver_args=driver_args):
    2549          #    return kinterbasdb.connect(**driver_args)
    2550
    2551          def connect(driver_args=driver_args):
    2552              return self.driver.connect(**driver_args)
    2553          self.pool_connection(connect)
    2554
    2555
    2556 -class InformixAdapter(BaseAdapter):
    2557
    2558      driver = globals().get('informixdb',None)
    2559
    2560      types = {
    2561          'boolean': 'CHAR(1)',
    2562          'string': 'VARCHAR(%(length)s)',
    2563          'text': 'BLOB SUB_TYPE 1',
    2564          'password': 'VARCHAR(%(length)s)',
    2565          'blob': 'BLOB SUB_TYPE 0',
    2566          'upload': 'VARCHAR(%(length)s)',
    2567          'integer': 'INTEGER',
    2568          'double': 'FLOAT',
    2569          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    2570          'date': 'DATE',
    2571          'time': 'CHAR(8)',
    2572          'datetime': 'DATETIME',
    2573          'id': 'SERIAL',
    2574          'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2575          'reference FK': 'REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s CONSTRAINT FK_%(table_name)s_%(field_name)s',
    2576          'reference TFK': 'FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s CONSTRAINT TFK_%(table_name)s_%(field_name)s',
    2577          'list:integer': 'BLOB SUB_TYPE 1',
    2578          'list:string': 'BLOB SUB_TYPE 1',
    2579          'list:reference': 'BLOB SUB_TYPE 1',
    2580          }
    2581
    2582 -    def RANDOM(self):
    2583          return 'Random()'
    2584
    2585 -    def NOT_NULL(self,default,field_type):
    2586          return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
    2587
    2588 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    2589          if limitby:
    2590              (lmin, lmax) = limitby
    2591              fetch_amt = lmax - lmin
    2592              dbms_version = int(self.connection.dbms_version.split('.')[0])
    2593              if lmin and (dbms_version >= 10):
    2594                  # Requires Informix 10.0+
    2595                  sql_s += ' SKIP %d' % (lmin, )
    2596              if fetch_amt and (dbms_version >= 9):
    2597                  # Requires Informix 9.0+
    2598                  sql_s += ' FIRST %d' % (fetch_amt, )
    2599          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    2600
    2601 -    def represent_exceptions(self, obj, fieldtype):
    2602          if fieldtype == 'date':
    2603              if isinstance(obj, (datetime.date, datetime.datetime)):
    2604                  obj = obj.isoformat()[:10]
    2605              else:
    2606                  obj = str(obj)
    2607              return "to_date('%s','%%Y-%%m-%%d')" % obj
    2608          elif fieldtype == 'datetime':
    2609              if isinstance(obj, datetime.datetime):
    2610                  obj = obj.isoformat()[:19].replace('T',' ')
    2611              elif isinstance(obj, datetime.date):
    2612                  obj = obj.isoformat()[:10]+' 00:00:00'
    2613              else:
    2614                  obj = str(obj)
    2615              return "to_date('%s','%%Y-%%m-%%d %%H:%%M:%%S')" % obj
    2616          return None
    2617
    2618 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2619                   credential_decoder=lambda x:x, driver_args={},
    2620                   adapter_args={}):
    2621          if not self.driver:
    2622              raise RuntimeError, "Unable to import driver"
    2623          self.db = db
    2624          self.dbengine = "informix"
    2625          self.uri = uri
    2626          self.pool_size = pool_size
    2627          self.folder = folder
    2628          self.db_codec = db_codec
    2629          self.find_or_make_work_folder()
    2630          uri = uri.split('://')[1]
    2631          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$').match(uri)
    2632          if not m:
    2633              raise SyntaxError, \
    2634                  "Invalid URI string in DAL: %s" % self.uri
    2635          user = credential_decoder(m.group('user'))
    2636          if not user:
    2637              raise SyntaxError, 'User required'
    2638          password = credential_decoder(m.group('password'))
    2639          if not password:
    2640              password = ''
    2641          host = m.group('host')
    2642          if not host:
    2643              raise SyntaxError, 'Host name required'
    2644          db = m.group('db')
    2645          if not db:
    2646              raise SyntaxError, 'Database name required'
    2647          user = credential_decoder(user)
    2648          password = credential_decoder(password)
    2649          dsn = '%s@%s' % (db,host)
    2650          driver_args.update(dict(user=user,password=password,autocommit=True))
    2651          def connect(dsn=dsn,driver_args=driver_args):
    2652              return self.driver.connect(dsn,**driver_args)
    2653          self.pool_connection(connect)
    2654
    2655 -    def execute(self,command):
    2656          if command[-1:]==';':
    2657              command = command[:-1]
    2658          return self.log_execute(command)
    2659
    2660 -    def lastrowid(self,table):
    2661          return self.cursor.sqlerrd[1]
    2662
    2663 -    def integrity_error_class(self):
    2664          return informixdb.IntegrityError
    2665
    2666
    2667 -class DB2Adapter(BaseAdapter):
    2668
    2669      driver = globals().get('pyodbc',None)
    2670
    2671      types = {
    2672          'boolean': 'CHAR(1)',
    2673          'string': 'VARCHAR(%(length)s)',
    2674          'text': 'CLOB',
    2675          'password': 'VARCHAR(%(length)s)',
    2676          'blob': 'BLOB',
    2677          'upload': 'VARCHAR(%(length)s)',
    2678          'integer': 'INT',
    2679          'double': 'DOUBLE',
    2680          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    2681          'date': 'DATE',
    2682          'time': 'TIME',
    2683          'datetime': 'TIMESTAMP',
    2684          'id': 'INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
    2685          'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2686          'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2687          'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
    2688          'list:integer': 'CLOB',
    2689          'list:string': 'CLOB',
    2690          'list:reference': 'CLOB',
    2691          }
    2692
    2693 -    def LEFT_JOIN(self):
    2694          return 'LEFT OUTER JOIN'
    2695
    2696 -    def RANDOM(self):
    2697          return 'RAND()'
    2698
    2699 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    2700          if limitby:
    2701              (lmin, lmax) = limitby
    2702              sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
    2703          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    2704
    2705 -    def represent_exceptions(self, obj, fieldtype):
    2706          if fieldtype == 'blob':
    2707              obj = base64.b64encode(str(obj))
    2708              return "BLOB('%s')" % obj
    2709          elif fieldtype == 'datetime':
    2710              if isinstance(obj, datetime.datetime):
    2711                  obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
    2712              elif isinstance(obj, datetime.date):
    2713                  obj = obj.isoformat()[:10]+'-00.00.00'
    2714              return "'%s'" % obj
    2715          return None
    2716
    2717 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2718                   credential_decoder=lambda x:x, driver_args={},
    2719                   adapter_args={}):
    2720          if not self.driver:
    2721              raise RuntimeError, "Unable to import driver"
    2722          self.db = db
    2723          self.dbengine = "db2"
    2724          self.uri = uri
    2725          self.pool_size = pool_size
    2726          self.folder = folder
    2727          self.db_codec = db_codec
    2728          self.find_or_make_work_folder()
    2729          cnxn = uri.split('://', 1)[1]
    2730          def connect(cnxn=cnxn,driver_args=driver_args):
    2731              return self.driver.connect(cnxn,**driver_args)
    2732          self.pool_connection(connect)
    2733
    2734 -    def execute(self,command):
    2735          if command[-1:]==';':
    2736              command = command[:-1]
    2737          return self.log_execute(command)
    2738
    2739 -    def lastrowid(self,table):
    2740          self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
    2741          return int(self.cursor.fetchone()[0])
    2742
    2743 -    def rowslice(self,rows,minimum=0,maximum=None):
    2744          if maximum is None:
    2745              return rows[minimum:]
    2746          return rows[minimum:maximum]
    2747
    2748
    2749 -class TeradataAdapter(BaseAdapter):
    2750
    2751      driver = globals().get('pyodbc',None)
    2752
    2753      types = {
    2754          'boolean': 'CHAR(1)',
    2755          'string': 'VARCHAR(%(length)s)',
    2756          'text': 'CLOB',
    2757          'password': 'VARCHAR(%(length)s)',
    2758          'blob': 'BLOB',
    2759          'upload': 'VARCHAR(%(length)s)',
    2760          'integer': 'INT',
    2761          'double': 'DOUBLE',
    2762          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    2763          'date': 'DATE',
    2764          'time': 'TIME',
    2765          'datetime': 'TIMESTAMP',
    2766          'id': 'INTEGER GENERATED ALWAYS AS IDENTITY',  # Teradata Specific
    2767          # Modified Constraint syntax for Teradata.
    2768          'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
    2769          'list:integer': 'CLOB',
    2770          'list:string': 'CLOB',
    2771          'list:reference': 'CLOB',
    2772          }
    2773
    2774 -    def LEFT_JOIN(self):
    2775          return 'LEFT OUTER JOIN'
    2776
    2777 -    def create_table(self, table,
    2778                       migrate=True,
    2779                       fake_migrate=False,
    2780                       polymodel=None):
    2781          fields = []
    2782          sql_fields = {}
    2783          sql_fields_aux = {}
    2784          TFK = {}
    2785          tablename = table._tablename
    2786          sortable = 0
    2787          for field in table:
    2788              sortable += 1
    2789              k = field.name
    2790              if isinstance(field.type,SQLCustomType):
    2791                  ftype = field.type.native or field.type.type
    2792              elif field.type.startswith('reference'):
    2793                  referenced = field.type[10:].strip()
    2794                  constraint_name = self.constraint_name(tablename, field.name)
    2795                  if hasattr(table,'_primarykey'):
    2796                      rtablename,rfieldname = referenced.split('.')
    2797                      rtable = table._db[rtablename]
    2798                      rfield = rtable[rfieldname]
    2799                      # must be PK reference or unique
    2800                      if rfieldname in rtable._primarykey or rfield.unique:
    2801                          ftype = self.types[rfield.type[:9]] % dict(length=rfield.length)
    2802                          # multicolumn primary key reference?
    2803                          if not rfield.unique and len(rtable._primarykey)>1 :
    2804                              # then it has to be a table level FK
    2805                              if rtablename not in TFK:
    2806                                  TFK[rtablename] = {}
    2807                              TFK[rtablename][rfieldname] = field.name
    2808                          else:
    2809                              ftype = ftype + \
    2810                                  self.types['reference FK'] %dict(\
    2811                                  constraint_name=constraint_name,
    2812                                  table_name=tablename,
    2813                                  field_name=field.name,
    2814                                  foreign_key='%s (%s)'%(rtablename, rfieldname),
    2815                                  on_delete_action=field.ondelete)
    2816                  else:
    2817                      # make a guess here for circular references
    2818                      id_fieldname = referenced in table._db and table._db[referenced]._id.name or 'id'
    2819                      ftype = self.types[field.type[:9]]\
    2820                          % dict(table_name=tablename,
    2821                                 field_name=field.name,
    2822                                 constraint_name=constraint_name,
    2823                                 foreign_key=referenced + ('(%s)' % id_fieldname),
    2824                                 on_delete_action=field.ondelete)
    2825              elif field.type.startswith('list:reference'):
    2826                  ftype = self.types[field.type[:14]]
    2827              elif field.type.startswith('decimal'):
    2828                  precision, scale = map(int,field.type[8:-1].split(','))
    2829                  ftype = self.types[field.type[:7]] % \
    2830                      dict(precision=precision,scale=scale)
    2831              elif not field.type in self.types:
    2832                  raise SyntaxError, 'Field: unknown field type: %s for %s' % \
    2833                      (field.type, field.name)
    2834              else:
    2835                  ftype = self.types[field.type]\
    2836                       % dict(length=field.length)
    2837              if not field.type.startswith('id') and not field.type.startswith('reference'):
    2838                  if field.notnull:
    2839                      ftype += ' NOT NULL'
    2840                  else:
    2841                      ftype += self.ALLOW_NULL()
    2842                  if field.unique:
    2843                      ftype += ' UNIQUE'
    2844
    2845              # add to list of fields
    2846              sql_fields[field.name] = dict(sortable=sortable,
    2847                                            type=str(field.type),
    2848                                            sql=ftype)
    2849
    2850              if isinstance(field.default,(str,int,float)):
    2851                  # Caveat: sql_fields and sql_fields_aux differ for default values.
    2852                  # sql_fields is used to trigger migrations and sql_fields_aux
    2853                  # is used for create tables.
    2854                  # The reason is that we do not want to trigger a migration simply
    2855                  # because a default value changes.
    2856                  not_null = self.NOT_NULL(field.default, field.type)
    2857                  ftype = ftype.replace('NOT NULL', not_null)
    2858              sql_fields_aux[field.name] = dict(sql=ftype)
    2859              fields.append('%s %s' % (field.name, ftype))
    2860          other = ';'
    2861
    2862          fields = ',\n    '.join(fields)
    2863          for rtablename in TFK:
    2864              rfields = TFK[rtablename]
    2865              pkeys = table._db[rtablename]._primarykey
    2866              fkeys = [ rfields[k] for k in pkeys ]
    2867              fields = fields + ',\n    ' + \
    2868                       self.types['reference TFK'] %\
    2869                       dict(table_name=tablename,
    2870                       field_name=', '.join(fkeys),
    2871                       foreign_table=rtablename,
    2872                       foreign_key=', '.join(pkeys),
    2873                       on_delete_action=field.ondelete)
    2874
    2875          if hasattr(table,'_primarykey'):
    2876              query = '''CREATE TABLE %s(\n    %s,\n    %s) %s''' % \
    2877                  (tablename, fields, self.PRIMARY_KEY(', '.join(table._primarykey)),other)
    2878          else:
    2879              query = '''CREATE TABLE %s(\n    %s\n)%s''' % \
    2880                  (tablename, fields, other)
    2881
    2882          if self.uri.startswith('sqlite:///'):
    2883              path_encoding = sys.getfilesystemencoding() or locale.getdefaultlocale()[1] or 'utf8'
    2884              dbpath = self.uri[9:self.uri.rfind('/')].decode('utf8').encode(path_encoding)
    2885          else:
    2886              dbpath = self.folder
    2887
    2888          if not migrate:
    2889              return query
    2890          elif self.uri.startswith('sqlite:memory'):
    2891              table._dbt = None
    2892          elif isinstance(migrate, str):
    2893              table._dbt = os.path.join(dbpath, migrate)
    2894          else:
    2895              table._dbt = os.path.join(dbpath, '%s_%s.table' \
    2896                                            % (table._db._uri_hash, tablename))
    2897          if table._dbt:
    2898              table._loggername = os.path.join(dbpath, 'sql.log')
    2899              logfile = self.file_open(table._loggername, 'a')
    2900          else:
    2901              logfile = None
    2902          if not table._dbt or not self.file_exists(table._dbt):
    2903              if table._dbt:
    2904                  logfile.write('timestamp: %s\n'
    2905                                 % datetime.datetime.today().isoformat())
    2906                  logfile.write(query + '\n')
    2907              if not fake_migrate:
    2908                  self.create_sequence_and_triggers(query,table)
    2909                  table._db.commit()
    2910              if table._dbt:
    2911                  tfile = self.file_open(table._dbt, 'w')
    2912                  cPickle.dump(sql_fields, tfile)
    2913                  self.file_close(tfile)
    2914                  if fake_migrate:
    2915                      logfile.write('faked!\n')
    2916                  else:
    2917                      logfile.write('success!\n')
    2918          else:
    2919              tfile = self.file_open(table._dbt, 'r')
    2920              try:
    2921                  sql_fields_old = cPickle.load(tfile)
    2922              except EOFError:
    2923                  self.file_close(tfile)
    2924                  self.file_close(logfile)
    2925                  raise RuntimeError, 'File %s appears corrupted' % table._dbt
    2926              self.file_close(tfile)
    2927              if sql_fields != sql_fields_old:
    2928                  self.migrate_table(table,
    2929                                     sql_fields, sql_fields_old,
    2930                                     sql_fields_aux, logfile,
    2931                                     fake_migrate=fake_migrate)
    2932          self.file_close(logfile)
    2933          return query
    2934
    2935 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    2936                   credential_decoder=lambda x:x, driver_args={},
    2937                   adapter_args={}):
    2938          if not self.driver:
    2939              raise RuntimeError, "Unable to import driver"
    2940          self.db = db
    2941          self.dbengine = "teradata"
    2942          self.uri = uri
    2943          self.pool_size = pool_size
    2944          self.folder = folder
    2945          self.db_codec = db_codec
    2946          self.find_or_make_work_folder()
    2947          cnxn = uri.split('://', 1)[1]
    2948          def connect(cnxn=cnxn,driver_args=driver_args):
    2949              return self.driver.connect(cnxn,**driver_args)
    2950          self.pool_connection(connect)
    2951
    2952      # Similar to MSSQL, Teradata can't specify a range (for Pageby)
    2953 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    2954          if limitby:
    2955              (lmin, lmax) = limitby
    2956              sql_s += ' TOP %i' % lmax
    2957          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    2958
    2959
    2960  INGRES_SEQNAME='ii***lineitemsequence' # NOTE invalid database object name
    2961                                         # (ANSI-SQL wants this form of name
    2962                                         # to be a delimited identifier)
    2963
    2964 -class IngresAdapter(BaseAdapter):
    2965
    2966      driver = globals().get('ingresdbi',None)
    2967
    2968      types = {
    2969          'boolean': 'CHAR(1)',
    2970          'string': 'VARCHAR(%(length)s)',
    2971          'text': 'CLOB',
    2972          'password': 'VARCHAR(%(length)s)',  ## Not sure what this contains utf8 or nvarchar. Or even bytes?
    2973          'blob': 'BLOB',
    2974          'upload': 'VARCHAR(%(length)s)',  ## FIXME utf8 or nvarchar... or blob? what is this type?
    2975          'integer': 'INTEGER4', # or int8...
    2976          'double': 'FLOAT8',
    2977          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    2978          'date': 'ANSIDATE',
    2979          'time': 'TIME WITHOUT TIME ZONE',
    2980          'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
    2981          'id': 'integer4 not null unique with default next value for %s' % INGRES_SEQNAME,
    2982          'reference': 'integer4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2983          'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    2984          'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO
    2985          'list:integer': 'CLOB',
    2986          'list:string': 'CLOB',
    2987          'list:reference': 'CLOB',
    2988          }
    2989
    2990 -    def LEFT_JOIN(self):
    2991          return 'LEFT OUTER JOIN'
    2992
    2993 -    def RANDOM(self):
    2994          return 'RANDOM()'
    2995
    2996 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    2997          if limitby:
    2998              (lmin, lmax) = limitby
    2999              fetch_amt = lmax - lmin
    3000              if fetch_amt:
    3001                  sql_s += ' FIRST %d ' % (fetch_amt, )
    3002              if lmin:
    3003                  # Requires Ingres 9.2+
    3004                  sql_o += ' OFFSET %d' % (lmin, )
    3005          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    3006
    3007 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    3008                   credential_decoder=lambda x:x, driver_args={},
    3009                   adapter_args={}):
    3010          if not self.driver:
    3011              raise RuntimeError, "Unable to import driver"
    3012          self.db = db
    3013          self.dbengine = "ingres"
    3014          self.uri = uri
    3015          self.pool_size = pool_size
    3016          self.folder = folder
    3017          self.db_codec = db_codec
    3018          self.find_or_make_work_folder()
    3019          connstr = self._uri.split(':', 1)[1]
    3020          # Simple URI processing
    3021          connstr = connstr.lstrip()
    3022          while connstr.startswith('/'):
    3023              connstr = connstr[1:]
    3024          database_name=connstr # Assume only (local) dbname is passed in
    3025          vnode = '(local)'
    3026          servertype = 'ingres'
    3027          trace = (0, None) # No tracing
    3028          driver_args.update(dict(database=database_name,
    3029                                     vnode=vnode,
    3030                                     servertype=servertype,
    3031                                     trace=trace))
    3032          def connect(driver_args=driver_args):
    3033              return self.driver.connect(**driver_args)
    3034          self.pool_connection(connect)
    3035
    3036 -    def create_sequence_and_triggers(self, query, table, **args):
    3037          # post create table auto inc code (if needed)
    3038          # modify table to btree for performance....
    3039          # Older Ingres releases could use rule/trigger like Oracle above.
    3040          if hasattr(table,'_primarykey'):
    3041              modify_tbl_sql = 'modify %s to btree unique on %s' % \
    3042                  (table._tablename,
    3043                   ', '.join(["'%s'" % x for x in table.primarykey]))
    3044              self.execute(modify_tbl_sql)
    3045          else:
    3046              tmp_seqname='%s_iisq' % table._tablename
    3047              query=query.replace(INGRES_SEQNAME, tmp_seqname)
    3048              self.execute('create sequence %s' % tmp_seqname)
    3049              self.execute(query)
    3050              self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
    3051
    3052
    3053 -    def lastrowid(self,table):
    3054          tmp_seqname='%s_iisq' % table
    3055          self.execute('select current value for %s' % tmp_seqname)
    3056          return int(self.cursor.fetchone()[0]) # don't really need int type cast here...
    3057
    3058 -    def integrity_error_class(self):
    3059          return ingresdbi.IntegrityError
    3060
    3061
    3062 -class IngresUnicodeAdapter(IngresAdapter):
    3063      types = {
    3064          'boolean': 'CHAR(1)',
    3065          'string': 'NVARCHAR(%(length)s)',
    3066          'text': 'NCLOB',
    3067          'password': 'NVARCHAR(%(length)s)',  ## Not sure what this contains utf8 or nvarchar. Or even bytes?
    3068          'blob': 'BLOB',
    3069          'upload': 'VARCHAR(%(length)s)',  ## FIXME utf8 or nvarchar... or blob? what is this type?
    3070          'integer': 'INTEGER4', # or int8...
    3071          'double': 'FLOAT8',
    3072          'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
    3073          'date': 'ANSIDATE',
    3074          'time': 'TIME WITHOUT TIME ZONE',
    3075          'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
    3076          'id': 'integer4 not null unique with default next value for %s'% INGRES_SEQNAME,
    3077          'reference': 'integer4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    3078          'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    3079          'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s', ## FIXME TODO
    3080          'list:integer': 'NCLOB',
    3081          'list:string': 'NCLOB',
    3082          'list:reference': 'NCLOB',
    3083          }
    3084
    3085 -class SAPDBAdapter(BaseAdapter):
    3086
    3087      driver = globals().get('sapdb',None)
    3088      support_distributed_transaction = False
    3089      types = {
    3090          'boolean': 'CHAR(1)',
    3091          'string': 'VARCHAR(%(length)s)',
    3092          'text': 'LONG',
    3093          'password': 'VARCHAR(%(length)s)',
    3094          'blob': 'LONG',
    3095          'upload': 'VARCHAR(%(length)s)',
    3096          'integer': 'INT',
    3097          'double': 'FLOAT',
    3098          'decimal': 'FIXED(%(precision)s,%(scale)s)',
    3099          'date': 'DATE',
    3100          'time': 'TIME',
    3101          'datetime': 'TIMESTAMP',
    3102          'id': 'INT PRIMARY KEY',
    3103          'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
    3104          'list:integer': 'LONG',
    3105          'list:string': 'LONG',
    3106          'list:reference': 'LONG',
    3107          }
    3108
    3109 -    def sequence_name(self,table):
    3110          return '%s_id_Seq' % table
    3111
    3112 -    def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
    3113          if limitby:
    3114              (lmin, lmax) = limitby
    3115              if len(sql_w) > 1:
    3116                  sql_w_row = sql_w + ' AND w_row > %i' % lmin
    3117              else:
    3118                  sql_w_row = 'WHERE w_row > %i' % lmin
    3119              return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
    3120          return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
    3121
    3122 -    def create_sequence_and_triggers(self, query, table, **args):
    3123          # following lines should only be executed if table._sequence_name does not exist
    3124          self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
    3125          self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
    3126                           % (table._tablename, table._id.name, table._sequence_name))
    3127          self.execute(query)
    3128
    3129 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    3130                   credential_decoder=lambda x:x, driver_args={},
    3131                   adapter_args={}):
    3132          if not self.driver:
    3133              raise RuntimeError, "Unable to import driver"
    3134          self.db = db
    3135          self.dbengine = "sapdb"
    3136          self.uri = uri
    3137          self.pool_size = pool_size
    3138          self.folder = folder
    3139          self.db_codec = db_codec
    3140          self.find_or_make_work_folder()
    3141          uri = uri.split('://')[1]
    3142          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$').match(uri)
    3143          if not m:
    3144              raise SyntaxError, "Invalid URI string in DAL"
    3145          user = credential_decoder(m.group('user'))
    3146          if not user:
    3147              raise SyntaxError, 'User required'
    3148          password = credential_decoder(m.group('password'))
    3149          if not password:
    3150              password = ''
    3151          host = m.group('host')
    3152          if not host:
    3153              raise SyntaxError, 'Host name required'
    3154          db = m.group('db')
    3155          if not db:
    3156              raise SyntaxError, 'Database name required'
    3157          def connect(user=user, password=password, database=db,
    3158                      host=host, driver_args=driver_args):
    3159              return self.driver.Connection(user, password, database,
    3160                                            host, **driver_args)
    3161          self.pool_connection(connect)
    3162
    3163 -    def lastrowid(self,table):
    3164          self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
    3165          return int(self.cursor.fetchone()[0])
    3166
    3167 -class CubridAdapter(MySQLAdapter):
    3168
    3169      driver = globals().get('cubriddb', None)
    3170
    3171 -    def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
    3172                   credential_decoder=lambda x:x, driver_args={},
    3173                   adapter_args={}):
    3174          if not self.driver:
    3175              raise RuntimeError, "Unable to import driver"
    3176          self.db = db
    3177          self.dbengine = "cubrid"
    3178          self.uri = uri
    3179          self.pool_size = pool_size
    3180          self.folder = folder
    3181          self.db_codec = db_codec
    3182          self.find_or_make_work_folder()
    3183          uri = uri.split('://')[1]
    3184          m = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$').match(uri)
    3185          if not m:
    3186              raise SyntaxError, \
    3187                  "Invalid URI string in DAL: %s" % self.uri
    3188          user = credential_decoder(m.group('user'))
    3189          if not user:
    3190              raise SyntaxError, 'User required'
    3191          password = credential_decoder(m.group('password'))
    3192          if not password:
    3193              password = ''
    3194          host = m.group('host')
    3195          if not host:
    3196              raise SyntaxError, 'Host name required'
    3197          db = m.group('db')
    3198          if not db:
    3199              raise SyntaxError, 'Database name required'
    3200          port = int(m.group('port') or '30000')
    3201          charset = m.group('charset') or 'utf8'
    3202          user = credential_decoder(user)
    3203          passwd = credential_decoder(password)
    3204          def connect(host=host,port=port,db=db,
    3205                      user=user,passwd=password,driver_args=driver_args):
    3206              return self.driver.connect(host,port,db,user,passwd,**driver_args)
    3207          self.pool_connection(connect)
    3208          self.execute('SET FOREIGN_KEY_CHECKS=1;')
    3209          self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
    3210
    3211
    3212  ######## GAE MySQL ##########
    3213
    3214 -class DatabaseStoredFile:
    3215
    3216      web2py_filesystem = False
    3217
    3218 -    def escape(self,obj):
    3219          return self.db._adapter.escape(obj)
    3220
    3221 -    def __init__(self,db,filename,mode):
    3222          if db._adapter.dbengine != 'mysql':
    3223              raise RuntimeError, "only MySQL can store metadata .table files in database for now"
    3224          self.db = db
    3225          self.filename = filename
    3226          self.mode = mode
    3227          if not self.web2py_filesystem:
    3228              self.db.executesql("CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(512), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;")
    3229              DatabaseStoredFile.web2py_filesystem = True
    3230          self.p=0
    3231          self.data = ''
    3232          if mode in ('r','rw','a'):
    3233              query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
    3234                  % filename
    3235              rows = self.db.executesql(query)
    3236              if rows:
    3237                  self.data = rows[0][0]
    3238              elif os.path.exists(filename):
    3239                  datafile = open(filename, 'r')
    3240                  try:
    3241                      self.data = datafile.read()
    3242                  finally:
    3243                      datafile.close()
    3244              elif mode in ('r','rw'):
    3245                  raise RuntimeError, "File %s does not exist" % filename
    3246
    3247 -    def read(self, bytes):
    3248          data = self.data[self.p:self.p+bytes]
    3249          self.p += len(data)
    3250          return data
    3251
    3252 -    def readline(self):
    3253          i = self.data.find('\n',self.p)+1
    3254          if i>0:
    3255              data, self.p = self.data[self.p:i], i
    3256          else:
    3257              data, self.p = self.data[self.p:], len(self.data)
    3258          return data
    3259
    3260 -    def write(self,data):
    3261          self.data += data
    3262
    3263 -    def close(self):
    3264          self.db.executesql("DELETE FROM web2py_filesystem WHERE path='%s'" \
    3265                                 % self.filename)
    3266          query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
    3267              % (self.filename, self.data.replace("'","''"))
    3268          self.db.executesql(query)
    3269          self.db.commit()
    3270
    3271      @staticmethod
    3272 -    def exists(db, filename):
    3273          if os.path.exists(filename):
    3274              return True
    3275          query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
    3276          if db.executesql(query):
    3277              return True
    3278          return False
    3279
    3280
    3281 -class UseDatabaseStoredFile:
    3282
    3283 -    def file_exists(self, filename):
    3284          return DatabaseStoredFile.exists(self.db,filename)
    3285
    3286 -    def file_open(self, filename, mode='rb', lock=True):
    3287          return DatabaseStoredFile(self.db,filename,mode)
    3288
    3289 -    def file_close(self, fileobj):
    3290          fileobj.close()
    3291
    3292 -    def file_delete(self,filename):
    3293          query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
    3294          self.db.executesql(query)
    3295          self.db.commit()
    3296
    3297 -class GoogleSQLAdapter(UseDatabaseStoredFile,MySQLAdapter):
    3298
    3299 -    def __init__(self, db, uri='google:sql://realm:domain/database',
    3300                   pool_size=0, folder=None, db_codec='UTF-8',
    3301                   credential_decoder=lambda x:x, driver_args={},
    3302                   adapter_args={}):
    3303
    3304          self.db = db
    3305          self.dbengine = "mysql"
    3306          self.uri = uri
    3307          self.pool_size = pool_size
    3308          self.folder = folder
    3309          self.db_codec = db_codec
    3310          self.folder = folder or '$HOME/'+thread.folder.split('/applications/',1)[1]
    3311
    3312          m = re.compile('^(?P<instance>.*)/(?P<db>.*)$').match(self.uri[len('google:sql://'):])
    3313          if not m:
    3314              raise SyntaxError, "Invalid URI string in SQLDB: %s" % self._uri
    3315          instance = credential_decoder(m.group('instance'))
    3316          db = credential_decoder(m.group('db'))
    3317          driver_args['instance'] = instance
    3318          createdb = adapter_args.get('createdb',True)
    3319          if not createdb:
    3320              driver_args['database'] = db
    3321          def connect(driver_args=driver_args):
    3322              return rdbms.connect(**driver_args)
    3323          self.pool_connection(connect)
    3324          if createdb:
    3325              # self.execute('DROP DATABASE %s' % db)
    3326              self.execute('CREATE DATABASE IF NOT EXISTS %s' % db)
    3327              self.execute('USE %s' % db)
    3328          self.execute("SET FOREIGN_KEY_CHECKS=1;")
    3329          self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
    3330
    3331 -class NoSQLAdapter(BaseAdapter):
    3332      can_select_for_update = False
    3333
    3334      @staticmethod
    3335 -    def to_unicode(obj):
    3336          if isinstance(obj, str):
    3337              return obj.decode('utf8')
    3338          elif not isinstance(obj, unicode):
    3339              return unicode(obj)
    3340          return obj
    3341
    3342 -    def represent(self, obj, fieldtype):
    3343          if isinstance(obj, CALLABLETYPES):
    3344              obj = obj()
    3345          if isinstance(fieldtype, SQLCustomType):
    3346              return fieldtype.encoder(obj)
    3347          if isinstance(obj, (Expression, Field)):
    3348              raise SyntaxError, "non supported on GAE"
    3349          if self.dbengine == 'google:datastore':
    3350              if isinstance(fieldtype, gae.Property):
    3351                  return obj
    3352          is_string = isinstance(fieldtype,str)
    3353          is_list = is_string and fieldtype.startswith('list:')
    3354          if is_list:
    3355              if not obj:
    3356                  obj = []
    3357              if not isinstance(obj, (list, tuple)):
    3358                  obj = [obj]
    3359          if obj == '' and not \
    3360                  (is_string and fieldtype[:2] in ['st','te','pa','up']):
    3361              return None
    3362          if not obj is None:
    3363              if isinstance(obj, list) and not is_list:
    3364                  obj = [self.represent(o, fieldtype) for o in obj]
    3365              elif fieldtype in ('integer','id'):
    3366                  obj = long(obj)
    3367              elif fieldtype == 'double':
    3368                  obj = float(obj)
    3369              elif is_string and fieldtype.startswith('reference'):
    3370                  if isinstance(obj, (Row, Reference)):
    3371                      obj = obj['id']
    3372                  obj = long(obj)
    3373              elif fieldtype == 'boolean':
    3374                  if obj and not str(obj)[0].upper() == 'F':
    3375                      obj = True
    3376                  else:
    3377                      obj = False
    3378              elif fieldtype == 'date':
    3379                  if not isinstance(obj, datetime.date):
    3380                      (y, m, d) = map(int,str(obj).strip().split('-'))
    3381                      obj = datetime.date(y, m, d)
    3382                  elif isinstance(obj,datetime.datetime):
    3383                      (y, m, d) = (obj.year, obj.month, obj.day)
    3384                      obj = datetime.date(y, m, d)
    3385              elif fieldtype == 'time':
    3386                  if not isinstance(obj, datetime.time):
    3387                      time_items = map(int,str(obj).strip().split(':')[:3])
    3388                      if len(time_items) == 3:
    3389                          (h, mi, s) = time_items
    3390                      else:
    3391                          (h, mi, s) = time_items + [0]
    3392                      obj = datetime.time(h, mi, s)
    3393              elif fieldtype == 'datetime':
    3394                  if not isinstance(obj, datetime.datetime):
    3395                      (y, m, d) = map(int,str(obj)[:10].strip().split('-'))
    3396                      time_items = map(int,str(obj)[11:].strip().split(':')[:3])
    3397                      while len(time_items)<3:
    3398                          time_items.append(0)
    3399                      (h, mi, s) = time_items
    3400                      obj = datetime.datetime(y, m, d, h, mi, s)
    3401              elif fieldtype == 'blob':
    3402                  pass
    3403              elif is_string and fieldtype.startswith('list:string'):
    3404                  return map(self.to_unicode,obj)
    3405              elif is_list:
    3406                  return map(int,obj)
    3407              else:
    3408                  obj = self.to_unicode(obj)
    3409          return obj
    3410
    3411 -    def _insert(self,table,fields):
    3412          return 'insert %s in %s' % (fields, table)
    3413
    3414 -    def _count(self,query,distinct=None):
    3415          return 'count %s' % repr(query)
    3416
    3417 -    def _select(self,query,fields,attributes):
    3418          return 'select %s where %s' % (repr(fields), repr(query))
    3419
    3420 -    def _delete(self,tablename, query):
    3421          return 'delete %s where %s' % (repr(tablename),repr(query))
    3422
    3423 -    def _update(self,tablename,query,fields):
    3424          return 'update %s (%s) where %s' % (repr(tablename),
    3425                                              repr(fields),repr(query))
    3426
    3427 -    def commit(self):
    3428          """
    3429          remember: no transactions on many NoSQL
    3430          """
    3431          pass
    3432
    3433 -    def rollback(self):
    3434          """
    3435          remember: no transactions on many NoSQL
    3436          """
    3437          pass
    3438
    3439 -    def close(self):
    3440          """
    3441          remember: no transactions on many NoSQL
    3442          """
    3443          pass
    3444
    3445
    3446      # these functions should never be called!
    3447 -    def OR(self,first,second): raise SyntaxError, "Not supported"
    3448 -    def AND(self,first,second): raise SyntaxError, "Not supported"
    3449 -    def AS(self,first,second): raise SyntaxError, "Not supported"
    3450 -    def ON(self,first,second): raise SyntaxError, "Not supported"
    3451 -    def STARTSWITH(self,first,second=None): raise SyntaxError, "Not supported"
    3452 -    def ENDSWITH(self,first,second=None): raise SyntaxError, "Not supported"
    3453 -    def ADD(self,first,second): raise SyntaxError, "Not supported"
    3454 -    def SUB(self,first,second): raise SyntaxError, "Not supported"
    3455 -    def MUL(self,first,second): raise SyntaxError, "Not supported"
    3456 -    def DIV(self,first,second): raise SyntaxError, "Not supported"
    3457 -    def LOWER(self,first): raise SyntaxError, "Not supported"
    3458 -    def UPPER(self,first): raise SyntaxError, "Not supported"
    3459 -    def EXTRACT(self,first,what): raise SyntaxError, "Not supported"
    3460 -    def AGGREGATE(self,first,what): raise SyntaxError, "Not supported"
    3461 -    def LEFT_JOIN(self): raise SyntaxError, "Not supported"
    3462 -    def RANDOM(self): raise SyntaxError, "Not supported"
    3463 -    def SUBSTRING(self,field,parameters):  raise SyntaxError, "Not supported"
    3464 -    def PRIMARY_KEY(self,key):  raise SyntaxError, "Not supported"
    3465 -    def ILIKE(self,first,second): raise SyntaxError, "Not supported"
    3466 -    def drop(self,table,mode):  raise SyntaxError, "Not supported"
    3467 -    def alias(self,table,alias): raise SyntaxError, "Not supported"
    3468 -    def migrate_table(self,*a,**b): raise SyntaxError, "Not supported"
    3469 -    def distributed_transaction_begin(self,key): raise SyntaxError, "Not supported"
    3470 -    def prepare(self,key): raise SyntaxError, "Not supported"
    3471 -    def commit_prepared(self,key): raise SyntaxError, "Not supported"
    3472 -    def rollback_prepared(self,key): raise SyntaxError, "Not supported"
    3473 -    def concat_add(self,table): raise SyntaxError, "Not supported"
    3474 -    def constraint_name(self, table, fieldname): raise SyntaxError, "Not supported"
    3475 -    def create_sequence_and_triggers(self, query, table, **args): pass
    3476 -    def log_execute(self,*a,**b): raise SyntaxError, "Not supported"
    3477 -    def execute(self,*a,**b): raise SyntaxError, "Not supported"
    3478 -    def represent_exceptions(self, obj, fieldtype): raise SyntaxError, "Not supported"
    3479 -    def lastrowid(self,table): raise SyntaxError, "Not supported"
    3480 -    def integrity_error_class(self): raise SyntaxError, "Not supported"
    3481 -    def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError, "Not supported"
    3482
    3483
    3484 -class GAEF(object):
    3485 -    def __init__(self,name,op,value,apply):
    3486          self.name=name=='id' and '__key__' or name
    3487          self.op=op
    3488          self.value=value
    3489          self.apply=apply
    3490 -    def __repr__(self):
    3491          return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
    3492
    3493 -class GoogleDatastoreAdapter(NoSQLAdapter):
    3494      uploads_in_blob = True
    3495      types = {}
    3496
    3497 -    def file_exists(self, filename): pass
    3498 -    def file_open(self, filename, mode='rb', lock=True): pass
    3499 -    def file_close(self, fileobj): pass
    3500
    3501 -    def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
    3502                   credential_decoder=lambda x:x, driver_args={},
    3503                   adapter_args={}):
    3504          self.types.update({
    3505                  'boolean': gae.BooleanProperty,
    3506                  'string': (lambda: gae.StringProperty(multiline=True)),
    3507                  'text': gae.TextProperty,
    3508                  'password': gae.StringProperty,
    3509                  'blob': gae.BlobProperty,
    3510                  'upload': gae.StringProperty,
    3511                  'integer': gae.IntegerProperty,
    3512                  'double': gae.FloatProperty,
    3513                  'decimal': GAEDecimalProperty,
    3514                  'date': gae.DateProperty,
    3515                  'time': gae.TimeProperty,
    3516                  'datetime': gae.DateTimeProperty,
    3517                  'id': None,
    3518                  'reference': gae.IntegerProperty,
    3519                  'list:string': (lambda: gae.StringListProperty(default=None)),
    3520                  'list:integer': (lambda: gae.ListProperty(int,default=None)),
    3521                  'list:reference': (lambda: gae.ListProperty(int,default=None)),
    3522          })
    3523          self.db = db
    3524          self.uri = uri
    3525          self.dbengine = 'google:datastore'
    3526          self.folder = folder
    3527          db['_lastsql'] = ''
    3528          self.db_codec = 'UTF-8'
    3529          self.pool_size = 0
    3530          match = re.compile('.*://(?P<namespace>.+)').match(uri)
    3531          if match:
    3532              namespace_manager.set_namespace(match.group('namespace'))
    3533
    3534 -    def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
    3535          myfields = {}
    3536          for k in table.fields:
    3537              if isinstance(polymodel,Table) and k in polymodel.fields():
    3538                  continue
    3539              field = table[k]
    3540              attr = {}
    3541              if isinstance(field.type, SQLCustomType):
    3542                  ftype = self.types[field.type.native or field.type.type](**attr)
    3543              elif isinstance(field.type, gae.Property):
    3544                  ftype = field.type
    3545              elif field.type.startswith('id'):
    3546                  continue
    3547              elif field.type.startswith('decimal'):
    3548                  precision, scale = field.type[7:].strip('()').split(',')
    3549                  precision = int(precision)
    3550                  scale = int(scale)
    3551                  ftype = GAEDecimalProperty(precision, scale, **attr)
    3552              elif field.type.startswith('reference'):
    3553                  if field.notnull:
    3554                      attr = dict(required=True)
    3555                  referenced = field.type[10:].strip()
    3556                  ftype = self.types[field.type[:9]](referenced)
    3557              elif field.type.startswith('list:reference'):
    3558                  if field.notnull:
    3559                      attr = dict(required=True)
    3560                  referenced = field.type[15:].strip()
    3561                  ftype = self.types[field.type[:14]](**attr)
    3562              elif field.type.startswith('list:'):
    3563                  ftype = self.types[field.type](**attr)
    3564              elif not field.type in self.types\
    3565                   or not self.types[field.type]:
    3566                  raise SyntaxError, 'Field: unknown field type: %s' % field.type
    3567              else:
    3568                  ftype = self.types[field.type](**attr)
    3569              myfields[field.name] = ftype
    3570          if not polymodel:
    3571              table._tableobj = classobj(table._tablename, (gae.Model, ), myfields)
    3572          elif polymodel==True:
    3573              table._tableobj = classobj(table._tablename, (PolyModel, ), myfields)
    3574          elif isinstance(polymodel,Table):
    3575              table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
    3576          else:
    3577              raise SyntaxError, "polymodel must be None, True, a table or a tablename"
    3578          return None
    3579
    3580 -    def expand(self,expression,field_type=None):
    3581          if isinstance(expression,Field):
    3582              if expression.type in ('text','blob'):
    3583                  raise SyntaxError, 'AppEngine does not index by: %s' % expression.type
    3584              return expression.name
    3585          elif isinstance(expression, (Expression, Query)):
    3586              if not expression.second is None:
    3587                  return expression.op(expression.first, expression.second)
    3588              elif not expression.first is None:
    3589                  return expression.op(expression.first)
    3590              else:
    3591                  return expression.op()
    3592          elif field_type:
    3593                  return self.represent(expression,field_type)
    3594          elif isinstance(expression,(list,tuple)):
    3595              return ','.join([self.represent(item,field_type) for item in expression])
    3596          else:
    3597              return str(expression)
    3598
    3599      ### TODO from gql.py Expression
    3600 -    def AND(self,first,second):
    3601          a = self.expand(first)
    3602          b = self.expand(second)
    3603          if b[0].name=='__key__' and a[0].name!='__key__':
    3604              return b+a
    3605          return a+b
    3606
    3607 -    def EQ(self,first,second=None):
    3608          if isinstance(second, Key):
    3609              return [GAEF(first.name,'=',second,lambda a,b:a==b)]
    3610          return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
    3611
    3612 -    def NE(self,first,second=None):
    3613          if first.type != 'id':
    3614              return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
    3615          else:
    3616              second = Key.from_path(first._tablename, long(second))
    3617              return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
    3618
    3619 -    def LT(self,first,second=None):
    3620          if first.type != 'id':
    3621              return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
    3622          else:
    3623              second = Key.from_path(first._tablename, long(second))
    3624              return [GAEF(first.name,'<',second,lambda a,b:a<b)]
    3625
    3626 -    def LE(self,first,second=None):
    3627          if first.type != 'id':
    3628              return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
    3629          else:
    3630              second = Key.from_path(first._tablename, long(second))
    3631              return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
    3632
    3633 -    def GT(self,first,second=None):
    3634          if first.type != 'id' or second==0 or second == '0':
    3635              return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
    3636          else:
    3637              second = Key.from_path(first._tablename, long(second))
    3638              return [GAEF(first.name,'>',second,lambda a,b:a>b)]
    3639
    3640 -    def GE(self,first,second=None):
    3641          if first.type != 'id':
    3642              return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
    3643          else:
    3644              second = Key.from_path(first._tablename, long(second))
    3645              return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
    3646
    3647 -    def INVERT(self,first):
    3648          return '-%s' % first.name
    3649
    3650 -    def COMMA(self,first,second):
    3651          return '%s, %s' % (self.expand(first),self.expand(second))
    3652
    3653 -    def BELONGS(self,first,second=None):
    3654          if not isinstance(second,(list, tuple)):
    3655              raise SyntaxError, "Not supported"
    3656          if first.type != 'id':
    3657              return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)]
    3658          else:
    3659              second = [Key.from_path(first._tablename, i) for i in second]
    3660              return [GAEF(first.name,'in',second,lambda a,b:a in b)]
    3661
    3662 -    def CONTAINS(self,first,second):
    3663          if not first.type.startswith('list:'):
    3664              raise SyntaxError, "Not supported"
    3665          return [GAEF(first.name,'=',self.expand(second,first.type[5:]),lambda a,b:a in b)]
    3666
    3667 -    def NOT(self,first):
    3668          nops = { self.EQ: self.NE,
    3669                   self.NE: self.EQ,
    3670                   self.LT: self.GE,
    3671                   self.GT: self.LE,
    3672                   self.LE: self.GT,
    3673                   self.GE: self.LT}
    3674          if not isinstance(first,Query):
    3675              raise SyntaxError, "Not suported"
    3676          nop = nops.get(first.op,None)
    3677          if not nop:
    3678              raise SyntaxError, "Not suported %s" % first.op.__name__
    3679          first.op = nop
    3680          return self.expand(first)
    3681
    3682 -    def truncate(self,table,mode):
    3683          self.db(table._id > 0).delete()
    3684
    3685 -    def select_raw(self,query,fields=None,attributes=None):
    3686          fields = fields or []
    3687          attributes = attributes or {}
    3688          new_fields = []
    3689          for item in fields:
    3690              if isinstance(item,SQLALL):
    3691                  new_fields += item.table
    3692              else:
    3693                  new_fields.append(item)
    3694          fields = new_fields
    3695          if query:
    3696              tablename = self.get_table(query)
    3697          elif fields:
    3698              tablename = fields[0].tablename
    3699              query = fields[0].table._id>0
    3700          else:
    3701              raise SyntaxError, "Unable to determine a tablename"
    3702
    3703          if query:
    3704              if use_common_filters(query):
    3705                  query = self.common_filter(query,[tablename])
    3706
    3707          tableobj = self.db[tablename]._tableobj
    3708          items = tableobj.all()
    3709          filters = self.expand(query)
    3710          for filter in filters:
    3711              if filter.name=='__key__' and filter.op=='>' and filter.value==0:
    3712                  continue
    3713              elif filter.name=='__key__' and filter.op=='=':
    3714                  if filter.value==0:
    3715                      items = []
    3716                  elif isinstance(filter.value, Key):
    3717                      item = tableobj.get(filter.value)
    3718                      items = (item and [item]) or []
    3719                  else:
    3720                      item = tableobj.get_by_id(filter.value)
    3721                      items = (item and [item]) or []
    3722              elif isinstance(items,list): # i.e. there is a single record!
    3723                  items = [i for i in items if filter.apply(getattr(item,filter.name),
    3724                                                            filter.value)]
    3725              else:
    3726                  if filter.name=='__key__': items.order('__key__')
    3727                  items = items.filter('%s %s' % (filter.name,filter.op),filter.value)
    3728          if not isinstance(items,list):
    3729              if attributes.get('left', None):
    3730                  raise SyntaxError, 'Set: no left join in appengine'
    3731              if attributes.get('groupby', None):
    3732                  raise SyntaxError, 'Set: no groupby in appengine'
    3733              orderby = attributes.get('orderby', False)
    3734              if orderby:
    3735                  ### THIS REALLY NEEDS IMPROVEMENT !!!
    3736                  if isinstance(orderby, (list, tuple)):
    3737                      orderby = xorify(orderby)
    3738                  if isinstance(orderby,Expression):
    3739                      orderby = self.expand(orderby)
    3740                  orders = orderby.split(', ')
    3741                  for order in orders:
    3742                      order={'-id':'-__key__','id':'__key__'}.get(order,order)
    3743                      items = items.order(order)
    3744              if attributes.get('limitby', None):
    3745                  (lmin, lmax) = attributes['limitby']
    3746                  (limit, offset) = (lmax - lmin, lmin)
    3747                  items = items.fetch(limit, offset=offset)
    3748          fields = self.db[tablename].fields
    3749          return (items, tablename, fields)
    3750
    3751 -    def select(self,query,fields,attributes):
    3752          (items, tablename, fields) = self.select_raw(query,fields,attributes)
    3753          # self.db['_lastsql'] = self._select(query,fields,attributes)
    3754          rows = [
    3755              [t=='id' and (int(item.key().id()) if item.key().id() else
    3756                            item.key().name()) or getattr(item, t) for t in fields]
    3757              for item in items]
    3758          colnames = ['%s.%s' % (tablename, t) for t in fields]
    3759          return self.parse(rows, fields, colnames, False)
    3760
    3761
    3762 -    def count(self,query,distinct=None):
    3763          if distinct:
    3764              raise RuntimeError, "COUNT DISTINCT not supported"
    3765          (items, tablename, fields) = self.select_raw(query)
    3766          # self.db['_lastsql'] = self._count(query)
    3767          try:
    3768              return len(items)
    3769          except TypeError:
    3770              return items.count(limit=None)
    3771
    3772 -    def delete(self,tablename, query):
    3773          """
    3774          This function was changed on 2010-05-04 because according to
    3775          http://code.google.com/p/googleappengine/issues/detail?id=3119
    3776          GAE no longer support deleting more than 1000 records.
    3777          """
    3778          # self.db['_lastsql'] = self._delete(tablename,query)
    3779          (items, tablename, fields) = self.select_raw(query)
    3780          # items can be one item or a query
    3781          if not isinstance(items,list):
    3782              counter = items.count(limit=None)
    3783              leftitems = items.fetch(1000)
    3784              while len(leftitems):
    3785                  gae.delete(leftitems)
    3786                  leftitems = items.fetch(1000)
    3787          else:
    3788              counter = len(items)
    3789              gae.delete(items)
    3790          return counter
    3791
    3792 -    def update(self,tablename,query,update_fields):
    3793          # self.db['_lastsql'] = self._update(tablename,query,update_fields)
    3794          (items, tablename, fields) = self.select_raw(query)
    3795          counter = 0
    3796          for item in items:
    3797              for field, value in update_fields:
    3798                  setattr(item, field.name, self.represent(value,field.type))
    3799              item.put()
    3800              counter += 1
    3801          logger.info(str(counter))
    3802          return counter
    3803
    3804 -    def insert(self,table,fields):
    3805          dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
    3806          # table._db['_lastsql'] = self._insert(table,fields)
    3807          tmp = table._tableobj(**dfields)
    3808          tmp.put()
    3809          rid = Reference(tmp.key().id())
    3810          (rid._table, rid._record) = (table, None)
    3811          return rid
    3812
    3813 -    def bulk_insert(self,table,items):
    3814          parsed_items = []
    3815          for item in items:
    3816              dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
    3817              parsed_items.append(table._tableobj(**dfields))
    3818          gae.put(parsed_items)
    3819          return True
    3820
    3821 -def uuid2int(uuidv):
    3822      return uuid.UUID(uuidv).int
    3823
    3824 -def int2uuid(n):
    3825      return str(uuid.UUID(int=n))
    3826
    3827 -class CouchDBAdapter(NoSQLAdapter):
    3828      uploads_in_blob = True
    3829      types = {
    3830                  'boolean': bool,
    3831                  'string': str,
    3832                  'text': str,
    3833                  'password': str,
    3834                  'blob': str,
    3835                  'upload': str,
    3836                  'integer': long,
    3837                  'double': float,
    3838                  'date': datetime.date,
    3839                  'time': datetime.time,
    3840                  'datetime': datetime.datetime,
    3841                  'id': long,
    3842                  'reference': long,
    3843                  'list:string': list,
    3844                  'list:integer': list,
    3845                  'list:reference': list,
    3846          }
    3847
    3848 -    def file_exists(self, filename): pass
    3849 -    def file_open(self, filename, mode='rb', lock=True): pass
    3850 -    def file_close(self, fileobj): pass
    3851
    3852 -    def expand(self,expression,field_type=None):
    3853          if isinstance(expression,Field):
    3854              if expression.type=='id':
    3855                  return "%s._id" % expression.tablename
    3856          return BaseAdapter.expand(self,expression,field_type)
    3857
    3858 -    def AND(self,first,second):
    3859          return '(%s && %s)' % (self.expand(first),self.expand(second))
    3860
    3861 -    def OR(self,first,second):
    3862          return '(%s || %s)' % (self.expand(first),self.expand(second))
    3863
    3864 -    def EQ(self,first,second):
    3865          if second is None:
    3866              return '(%s == null)' % self.expand(first)
    3867          return '(%s == %s)' % (self.expand(first),self.expand(second,first.type))
    3868
    3869 -    def NE(self,first,second):
    3870          if second is None:
    3871              return '(%s != null)' % self.expand(first)
    3872          return '(%s != %s)' % (self.expand(first),self.expand(second,first.type))
    3873
    3874 -    def COMMA(self,first,second):
    3875          return '%s + %s' % (self.expand(first),self.expand(second))
    3876
    3877 -    def represent(self, obj, fieldtype):
    3878          value = NoSQLAdapter.represent(self, obj, fieldtype)
    3879          if fieldtype=='id':
    3880              return repr(str(int(value)))
    3881          elif fieldtype in ('date','time','datetime','boolean'):
    3882              return serializers.json(value)
    3883          return repr(not isinstance(value,unicode) and value or value.encode('utf8'))
    3884
    3885 -    def __init__(self,db,uri='couchdb://127.0.0.1:5984',
    3886                   pool_size=0,folder=None,db_codec ='UTF-8',
    3887                   credential_decoder=lambda x:x, driver_args={},
    3888                   adapter_args={}):
    3889          self.db = db
    3890          self.uri = uri
    3891          self.dbengine = 'couchdb'
    3892          self.folder = folder
    3893          db['_lastsql'] = ''
    3894          self.db_codec = 'UTF-8'
    3895          self.pool_size = pool_size
    3896
    3897          url='http://'+uri[10:]
    3898          def connect(url=url,driver_args=driver_args):
    3899              return couchdb.Server(url,**driver_args)
    3900          self.pool_connection(connect,cursor=False)
    3901
    3902 -    def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
    3903          if migrate:
    3904              try:
    3905                  self.connection.create(table._tablename)
    3906              except:
    3907                  pass
    3908
    3909 -    def insert(self,table,fields):
    3910          id = uuid2int(web2py_uuid())
    3911          ctable = self.connection[table._tablename]
    3912          values = dict((k.name,self.represent(v,k.type)) for k,v in fields)
    3913          values['_id'] = str(id)
    3914          ctable.save(values)
    3915          return id
    3916
    3917 -    def _select(self,query,fields,attributes):
    3918          if not isinstance(query,Query):
    3919              raise SyntaxError, "Not Supported"
    3920          for key in set(attributes.keys())-set(('orderby','groupby','limitby',
    3921                                                 'required','cache','left',
    3922                                                 'distinct','having')):
    3923              raise SyntaxError, 'invalid select attribute: %s' % key
    3924          new_fields=[]
    3925          for item in fields:
    3926              if isinstance(item,SQLALL):
    3927                  new_fields += item.table
    3928              else:
    3929                  new_fields.append(item)
    3930          def uid(fd):
    3931              return fd=='id' and '_id' or fd
    3932          def get(row,fd):
    3933              return fd=='id' and int(row['_id']) or row.get(fd,None)
    3934          fields = new_fields
    3935          tablename = self.get_table(query)
    3936          fieldnames = [f.name for f in (fields or self.db[tablename])]
    3937          colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
    3938          fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
    3939          fn="function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);}" %\
    3940              dict(t=tablename,
    3941                   query=self.expand(query),
    3942                   order='%s._id' % tablename,
    3943                   fields=fields)
    3944          return fn, colnames
    3945
    3946 -    def select(self,query,fields,attributes):
    3947          if not isinstance(query,Query):
    3948              raise SyntaxError, "Not Supported"
    3949          fn, colnames = self._select(query,fields,attributes)
    3950          tablename = colnames[0].split('.')[0]
    3951          ctable = self.connection[tablename]
    3952          rows = [cols['value'] for cols in ctable.query(fn)]
    3953          return self.parse(rows, fields, colnames, False)
    3954
    3955 -    def delete(self,tablename,query):
    3956          if not isinstance(query,Query):
    3957              raise SyntaxError, "Not Supported"
    3958          if query.first.type=='id' and query.op==self.EQ:
    3959              id = query.second
    3960              tablename = query.first.tablename
    3961              assert(tablename == query.first.tablename)
    3962              ctable = self.connection[tablename]
    3963              try:
    3964                  del ctable[str(id)]
    3965                  return 1
    3966              except couchdb.http.ResourceNotFound:
    3967                  return 0
    3968          else:
    3969              tablename = self.get_table(query)
    3970              rows = self.select(query,[self.db[tablename]._id],{})
    3971              ctable = self.connection[tablename]
    3972              for row in rows:
    3973                  del ctable[str(row.id)]
    3974              return len(rows)
    3975
    3976 -    def update(self,tablename,query,fields):
    3977          if not isinstance(query,Query):
    3978              raise SyntaxError, "Not Supported"
    3979          if query.first.type=='id' and query.op==self.EQ:
    3980              id = query.second
    3981              tablename = query.first.tablename
    3982              ctable = self.connection[tablename]
    3983              try:
    3984                  doc = ctable[str(id)]
    3985                  for key,value in fields:
    3986                      doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
    3987                  ctable.save(doc)
    3988                  return 1
    3989              except couchdb.http.ResourceNotFound:
    3990                  return 0
    3991          else:
    3992              tablename = self.get_table(query)
    3993              rows = self.select(query,[self.db[tablename]._id],{})
    3994              ctable = self.connection[tablename]
    3995              table = self.db[tablename]
    3996              for row in rows:
    3997                  doc = ctable[str(row.id)]
    3998                  for key,value in fields:
    3999                      doc[key.name] = self.represent(value,table[key.name].type)
    4000                  ctable.save(doc)
    4001              return len(rows)
    4002
    4003 -    def count(self,query,distinct=None):
    4004          if distinct:
    4005              raise RuntimeError, "COUNT DISTINCT not supported"
    4006          if not isinstance(query,Query):
    4007              raise SyntaxError, "Not Supported"
    4008          tablename = self.get_table(query)
    4009          rows = self.select(query,[self.db[tablename]._id],{})
    4010          return len(rows)
    4011
    4012 -def cleanup(text):
    4013      """
    4014      validates that the given text is clean: only contains [0-9a-zA-Z_]
    4015      """
    4016
    4017      if re.compile('[^0-9a-zA-Z_]').findall(text):
    4018          raise SyntaxError, \
    4019              'only [0-9a-zA-Z_] allowed in table and field names, received %s' \
    4020              % text
    4021      return text
    4022
    4023 -class MongoDBAdapter(NoSQLAdapter):
    4024      uploads_in_blob = True
    4025
    4026      types = {
    4027                  'boolean': bool,
    4028                  'string': str,
    4029                  'text': str,
    4030                  'password': str,
    4031                  'blob': str,
    4032                  'upload': str,
    4033                  'integer': long,
    4034                  'double': float,
    4035                  'date': datetime.date,
    4036                  'time': datetime.time,
    4037                  'datetime': datetime.datetime,
    4038                  'id': long,
    4039                  'reference': long,
    4040                  'list:string': list,
    4041                  'list:integer': list,
    4042                  'list:reference': list,
    4043          }
    4044
    4045 -    def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
    4046                   pool_size=0,folder=None,db_codec ='UTF-8',
    4047                   credential_decoder=lambda x:x, driver_args={},
    4048                   adapter_args={}):
    4049          m=None
    4050          try:
    4051              #Since version 2
    4052              import pymongo.uri_parser
    4053              m = pymongo.uri_parser.parse_uri(uri)
    4054          except ImportError:
    4055              try:
    4056                  #before version 2 of pymongo
    4057                  import pymongo.connection
    4058                  m = pymongo.connection._parse_uri(uri)
    4059              except ImportError:
    4060                  raise ImportError("Uriparser for mongodb is not available")
    4061          except:
    4062              raise SyntaxError("This type of uri is not supported by the mongodb uri parser")
    4063          self.db = db
    4064          self.uri = uri
    4065          self.dbengine = 'mongodb'
    4066          self.folder = folder
    4067          db['_lastsql'] = ''
    4068          self.db_codec = 'UTF-8'
    4069          self.pool_size = pool_size
    4070          #this is the minimum amount of replicates that it should wait for on insert/update
    4071          self.minimumreplication = adapter_args.get('minimumreplication',0)
    4072          #by default alle insert and selects are performand asynchronous, but now the default is
    4073          #synchronous, except when overruled by either this default or function parameter
    4074          self.safe = adapter_args.get('safe',True)
    4075
    4076
    4077          if isinstance(m,tuple):
    4078              m = {"database" : m[1]}
    4079          if m.get('database')==None:
    4080              raise SyntaxError("Database is required!")
    4081          def connect(uri=self.uri,m=m):
    4082              try:
    4083                  return pymongo.Connection(uri)[m.get('database')]
    4084              except pymongo.errors.ConnectionFailure, inst:
    4085                  raise SyntaxError, "The connection to " + uri + " could not be made"
    4086              except Exception, inst:
    4087                  if inst == "cannot specify database without a username and password":
    4088                      raise SyntaxError("You are probebly running version 1.1 of pymongo which contains a bug which requires authentication. Update your pymongo.")
    4089                  else:
    4090                      raise SyntaxError("This is not an official Mongodb uri (http://www.mongodb.org/display/DOCS/Connections) Error : %s" % inst)
    4091          self.pool_connection(connect,cursor=False)
    4092
    4093
    4094
    4095 -    def represent(self, obj, fieldtype):
    4096          value = NoSQLAdapter.represent(self, obj, fieldtype)
    4097          if fieldtype  =='date':
    4098              if value == None:
    4099                  return value
    4100              t = datetime.time(0, 0, 0)#this piece of data can be stripped of based on the fieldtype
    4101              return datetime.datetime.combine(value, t) #mongodb doesn't has a date object and so it must datetime, string or integer
    4102          elif fieldtype == 'time':
    4103              if value == None:
    4104                  return value
    4105              d = datetime.date(2000, 1, 1) #this piece of data can be stripped of based on the fieldtype
    4106              return datetime.datetime.combine(d, value) #mongodb doesn't has a  time object and so it must datetime, string or integer
    4107          elif fieldtype == 'list:string' or fieldtype == 'list:integer' or fieldtype == 'list:reference':
    4108              return value #raise SyntaxError, "Not Supported"
    4109          return value
    4110
    4111      #Safe determines whether a asynchronious request is done or a synchronious action is done
    4112      #For safety, we use by default synchronious requests
    4113 -    def insert(self,table,fields,safe=None):
    4114          if safe==None:
    4115              safe=self.safe
    4116          ctable = self.connection[table._tablename]
    4117          values = dict((k.name,self.represent(v,table[k.name].type)) for k,v in fields)
    4118          ctable.insert(values,safe=safe)
    4119          return int(str(values['_id']), 16)
    4120
    4121 -    def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None, isCapped=False):
    4122          if isCapped:
    4123              raise RuntimeError, "Not implemented"
    4124          else:
    4125              pass
    4126
    4127 -    def count(self,query,distinct=None,snapshot=True):
    4128          if distinct:
    4129              raise RuntimeError, "COUNT DISTINCT not supported"
    4130          if not isinstance(query,Query):
    4131              raise SyntaxError, "Not Supported"
    4132          tablename = self.get_table(query)
    4133          return int(self.select(query,[self.db[tablename]._id],{},count=True,snapshot=snapshot)['count'])
    4134          #Maybe it would be faster if we just implemented the pymongo .count() function which is probably quicker?
    4135          # therefor call __select() connection[table].find(query).count() Since this will probably reduce the return set?
    4136
    4137 -    def expand(self, expression, field_type=None):
    4138          import pymongo.objectid
    4139          #if isinstance(expression,Field):
    4140          #    if expression.type=='id':
    4141          #        return {_id}"
    4142          if isinstance(expression, Query):
    4143              print "in expand and this is a query"
    4144              # any query using 'id':=
    4145              #   set name as _id (as per pymongo/mongodb primary key)
    4146              #   convert second arg to an objectid field (if its not already)
    4147              #   if second arg is 0 convert to objectid
    4148              if isinstance(expression.first,Field) and expression.first.type == 'id':
    4149                  expression.first.name = '_id'
    4150                  if expression.second != 0 and not isinstance(expression.second,pymongo.objectid.ObjectId):
    4151                      if isinstance(expression.second,int):
    4152                          try:
    4153                              #Cause the reference field is by default an integer and therefor this must be an integer to be able to work with other databases
    4154                              expression.second = pymongo.objectid.ObjectId(("%X" % expression.second))
    4155                          except:
    4156                              raise SyntaxError, 'The second argument must by an integer that can represent an objectid.'
    4157                      else:
    4158                          try:
    4159                              #But a direct id is also possible
    4160                              expression.second = pymongo.objectid.ObjectId(expression.second)
    4161                          except:
    4162                              raise SyntaxError, 'second argument must be of type bson.objectid.ObjectId or an objectid representable integer'
    4163                  elif expression.second == 0:
    4164                      expression.second = pymongo.objectid.ObjectId('000000000000000000000000')
    4165                  return expression.op(expression.first, expression.second)
    4166          if isinstance(expression, Field):
    4167              if expression.type=='id':
    4168                  return "_id"
    4169              else:
    4170                  return expression.name
    4171              #return expression
    4172          elif isinstance(expression, (Expression, Query)):
    4173              if not expression.second is None:
    4174                  return expression.op(expression.first, expression.second)
    4175              elif not expression.first is None:
    4176                  return expression.op(expression.first)
    4177              elif not isinstance(expression.op, str):
    4178                  return expression.op()
    4179              else:
    4180                  return expression.op
    4181          elif field_type:
    4182              return str(self.represent(expression,field_type))
    4183          elif isinstance(expression,(list,tuple)):
    4184              return ','.join(self.represent(item,field_type) for item in expression)
    4185          else:
    4186              return expression
    4187
    4188 -    def _select(self,query,fields,attributes):
    4189          from pymongo import son
    4190
    4191          for key in set(attributes.keys())-set(('limitby','orderby')):
    4192              raise SyntaxError, 'invalid select attribute: %s' % key
    4193
    4194          new_fields=[]
    4195          mongosort_list = []
    4196
    4197          # try an orderby attribute
    4198          orderby = attributes.get('orderby', False)
    4199          limitby = attributes.get('limitby', False)
    4200          #distinct = attributes.get('distinct', False)
    4201          if orderby:
    4202              #print "in if orderby %s" % orderby
    4203              if isinstance(orderby, (list, tuple)):
    4204                  print "in xorify"
    4205                  orderby = xorify(orderby)
    4206
    4207
    4208              # !!!! need to add 'random'
    4209              for f in self.expand(orderby).split(','):
    4210                  if f.startswith('-'):
    4211                      mongosort_list.append((f[1:],-1))
    4212                  else:
    4213                      mongosort_list.append((f,1))
    4214              print "mongosort_list = %s" % mongosort_list
    4215
    4216          if limitby:
    4217              # a tuple
    4218              limitby_skip,limitby_limit = limitby
    4219          else:
    4220              limitby_skip = 0
    4221              limitby_limit = 0
    4222
    4223
    4224
    4225
    4226          #if distinct:
    4227              #print "in distinct %s" % distinct
    4228
    4229          mongofields_dict = son.SON()
    4230          mongoqry_dict = {}
    4231          for item in fields:
    4232              if isinstance(item,SQLALL):
    4233                  new_fields += item.table
    4234              else:
    4235                  new_fields.append(item)
    4236          fields = new_fields
    4237          if isinstance(query,Query):
    4238              tablename = self.get_table(query)
    4239          elif len(fields) != 0:
    4240              tablename = fields[0].tablename
    4241          else:
    4242              raise SyntaxError, "The table name could not be found in the query nor from the select statement."
    4243          fieldnames = [f for f in (fields or self.db[tablename])]  # ie table.field
    4244          mongoqry_dict = self.expand(query)
    4245          for f in fieldnames:
    4246              mongofields_dict[f.name] = 1  # ie field=1
    4247          return tablename, mongoqry_dict, mongofields_dict, mongosort_list, limitby_limit, limitby_skip
    4248
    4249      # need to define all the 'sql' methods gt,lt etc....
    4250
    4251 -    def select(self,query,fields,attributes,count=False,snapshot=False):
    4252          withId=False
    4253          tablename, mongoqry_dict , mongofields_dict, mongosort_list, limitby_limit, limitby_skip = self._select(query,fields,attributes)
    4254          for key in mongofields_dict.keys():
    4255              if key == 'id':
    4256                  withId =  True
    4257                  break;
    4258          try:
    4259              print "mongoqry_dict=%s" % mongoqry_dict
    4260          except:
    4261              pass
    4262          print "mongofields_dict=%s" % mongofields_dict
    4263          ctable = self.connection[tablename]
    4264          if count:
    4265              return {'count' : ctable.find(mongoqry_dict,mongofields_dict,skip=limitby_skip, limit=limitby_limit, sort=mongosort_list,snapshot=snapshot).count()}
    4266          else:
    4267              mongo_list_dicts = ctable.find(mongoqry_dict,mongofields_dict,skip=limitby_skip, limit=limitby_limit, sort=mongosort_list,snapshot=snapshot) # pymongo cursor object
    4268          print "mongo_list_dicts=%s" % mongo_list_dicts
    4269          #if mongo_list_dicts.count() > 0: #
    4270              #colnames = mongo_list_dicts[0].keys() # assuming all docs have same "shape", grab colnames from first dictionary (aka row)
    4271          #else:
    4272              #colnames = mongofields_dict.keys()
    4273          #print "colnames = %s" % colnames
    4274          #rows = [row.values() for row in mongo_list_dicts]
    4275          rows = []
    4276          for record in mongo_list_dicts:
    4277              row=[]
    4278              for column in record:
    4279                  if withId and (column == '_id'):
    4280                      if isinstance(record[column],pymongo.objectid.ObjectId):
    4281                          row.append( int(str(record[column]),16))
    4282                      else:
    4283                          #in case of alternative key
    4284                          row.append( record[column] )
    4285                  elif not (column == '_id'):
    4286                      row.append(record[column])
    4287              rows.append(row)
    4288                      #else the id is not supposed to be included. Work around error. mongo always sends key:(
    4289
    4290          return self.parse(rows,fields,mongofields_dict.keys(),False)
    4291
    4292 -    def INVERT(self,first):
    4293          #print "in invert first=%s" % first
    4294          return '-%s' % self.expand(first)
    4295
    4296 -    def drop(self, table, mode=''):
    4297          ctable = self.connection[table._tablename]
    4298          ctable.drop()
    4299
    4300
    4301 -    def truncate(self,table,mode,safe=None):
    4302          if safe==None:
    4303              safe=self.safe
    4304          ctable = self.connection[table._tablename]
    4305          ctable.remove(None, safe=True)
    4306
    4307      #the update function should return a string
    4308 -    def oupdate(self,tablename,query,fields):
    4309          if not isinstance(query,Query):
    4310              raise SyntaxError, "Not Supported"
    4311          filter = None
    4312          if query:
    4313              filter = self.expand(query)
    4314          f_v = []
    4315
    4316
    4317          modify = { '$set' : dict(((k.name,self.represent(v,k.type)) for k,v in fields)) }
    4318          return modify,filter
    4319
    4320      #TODO implement update
    4321      #TODO implement set operator
    4322      #TODO implement find and modify
    4323      #todo implement complex update
    4324 -    def update(self,tablename,query,fields,safe=None):
    4325          if safe==None:
    4326              safe=self.safe
    4327          #return amount of adjusted rows or zero, but no exceptions related not finding the result
    4328          if not isinstance(query,Query):
    4329              raise RuntimeError, "Not implemented"
    4330          amount = self.count(query,False)
    4331          modify,filter = self.oupdate(tablename,query,fields)
    4332          try:
    4333              if safe:
    4334                  return self.connection[tablename].update(filter,modify,multi=True,safe=safe).n
    4335              else:
    4336                  amount =self.count(query)
    4337                  self.connection[tablename].update(filter,modify,multi=True,safe=safe)
    4338                  return amount
    4339          except:
    4340              #TODO Reverse update query to verifiy that the query succeded
    4341              return 0
    4342      """
    4343      An special update operator that enables the update of specific field
    4344      return a dict
    4345      """
    4346
    4347
    4348
    4349      #this function returns a dict with the where clause and update fields
    4350 -    def _update(self,tablename,query,fields):
    4351          return str(self.oupdate(tablename,query,fields))
    4352
    4353 -    def bulk_insert(self, table, items):
    4354          return [self.insert(table,item) for item in items]
    4355
    4356      #TODO This will probably not work:(
    4357 -    def NOT(self, first):
    4358          result = {}
    4359          result["$not"] = self.expand(first)
    4360          return result
    4361
    4362 -    def AND(self,first,second):
    4363          f = self.expand(first)
    4364          s = self.expand(second)
    4365          f.update(s)
    4366          return f
    4367
    4368 -    def OR(self,first,second):
    4369          # pymongo expects: .find( {'$or' : [{'name':'1'}, {'name':'2'}] } )
    4370          result = {}
    4371          f = self.expand(first)
    4372          s = self.expand(second)
    4373          result['$or'] = [f,s]
    4374          return result
    4375
    4376 -    def BELONGS(self, first, second):
    4377          if isinstance(second, str):
    4378              return {self.expand(first) : {"$in" : [ second[:-1]]} }
    4379          elif second==[] or second==():
    4380              return {1:0}
    4381          items = [self.expand(item, first.type) for item in second]
    4382          return {self.expand(first) : {"$in" : items} }
    4383
    4384 -    def LIKE(self, first, second):
    4385          #escaping regex operators?
    4386          return {self.expand(first) : ('%s' % self.expand(second, 'string').replace('%','/'))}
    4387
    4388 -    def STARTSWITH(self, first, second):
    4389          #escaping regex operators?
    4390          return {self.expand(first) : ('/^%s/' % self.expand(second, 'string'))}
    4391
    4392 -    def ENDSWITH(self, first, second):
    4393          #escaping regex operators?
    4394          return {self.expand(first) : ('/%s^/' % self.expand(second, 'string'))}
    4395
    4396 -    def CONTAINS(self, first, second):
    4397          #There is a technical difference, but mongodb doesn't support that, but the result will be the same
    4398          return {self.expand(first) : ('/%s/' % self.expand(second, 'string'))}
    4399
    4400 -    def EQ(self,first,second):
    4401          result = {}
    4402          #if second is None:
    4403              #return '(%s == null)' % self.expand(first)
    4404          #return '(%s == %s)' % (self.expand(first),self.expand(second,first.type))
    4405          result[self.expand(first)] = self.expand(second)
    4406          return result
    4407
    4408 -    def NE(self, first, second=None):
    4409          print "in NE"
    4410          result = {}
    4411          result[self.expand(first)] = {'$ne': self.expand(second)}
    4412          return result
    4413
    4414 -    def LT(self,first,second=None):
    4415          if second is None:
    4416              raise RuntimeError, "Cannot compare %s < None" % first
    4417          print "in LT"
    4418          result = {}
    4419          result[self.expand(first)] = {'$lt': self.expand(second)}
    4420          return result
    4421
    4422 -    def LE(self,first,second=None):
    4423          if second is None:
    4424              raise RuntimeError, "Cannot compare %s <= None" % first
    4425          print "in LE"
    4426          result = {}
    4427          result[self.expand(first)] = {'$lte': self.expand(second)}
    4428          return result
    4429
    4430 -    def GT(self,first,second):
    4431          print "in GT"
    4432          #import pymongo.objectid
    4433          result = {}
    4434          #if expanded_first == '_id':
    4435              #if expanded_second != 0 and not isinstance(second,pymongo.objectid.ObjectId):
    4436                  #raise SyntaxError, 'second argument must be of type bson.objectid.ObjectId'
    4437              #elif expanded_second == 0:
    4438                  #expanded_second = pymongo.objectid.ObjectId('000000000000000000000000')
    4439          result[self.expand(first)] = {'$gt': self.expand(second)}
    4440          return result
    4441
    4442 -    def GE(self,first,second=None):
    4443          if second is None:
    4444              raise RuntimeError, "Cannot compare %s >= None" % first
    4445          print "in GE"
    4446          result = {}
    4447          result[self.expand(first)] = {'$gte': self.expand(second)}
    4448          return result
    4449
    4450 -    def ADD(self, first, second):
    4451          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4452          return '%s + %s' % (self.expand(first), self.expand(second, first.type))
    4453
    4454 -    def SUB(self, first, second):
    4455          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4456          return '(%s - %s)' % (self.expand(first), self.expand(second, first.type))
    4457
    4458 -    def MUL(self, first, second):
    4459          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4460          return '(%s * %s)' % (self.expand(first), self.expand(second, first.type))
    4461
    4462 -    def DIV(self, first, second):
    4463          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4464          return '(%s / %s)' % (self.expand(first), self.expand(second, first.type))
    4465
    4466 -    def MOD(self, first, second):
    4467          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4468          return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type))
    4469
    4470 -    def AS(self, first, second):
    4471          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4472          return '%s AS %s' % (self.expand(first), second)
    4473
    4474      #We could implement an option that simulates a full featured SQL database. But I think the option should be set explicit or implemented as another library.
    4475 -    def ON(self, first, second):
    4476          raise NotImplementedError, "This is not possible in NoSQL, but can be simulated with a wrapper."
    4477          return '%s ON %s' % (self.expand(first), self.expand(second))
    4478
    4479 -    def COMMA(self, first, second):
    4480          return '%s, %s' % (self.expand(first), self.expand(second))
    4481
    4482 -    def bulk_insert(self, table, items):
    4483          return [self.insert(table,item) for item in items]
    4484
    4485      #TODO This will probably not work:(
    4486 -    def NOT(self, first):
    4487          result = {}
    4488          result["$not"] = self.expand(first)
    4489          return result
    4490
    4491 -    def AND(self,first,second):
    4492          f = self.expand(first)
    4493          s = self.expand(second)
    4494          f.update(s)
    4495          return f
    4496
    4497 -    def OR(self,first,second):
    4498          # pymongo expects: .find( {'$or' : [{'name':'1'}, {'name':'2'}] } )
    4499          result = {}
    4500          f = self.expand(first)
    4501          s = self.expand(second)
    4502          result['$or'] = [f,s]
    4503          return result
    4504
    4505 -    def BELONGS(self, first, second):
    4506          if isinstance(second, str):
    4507              return {self.expand(first) : {"$in" : [ second[:-1]]} }
    4508          elif second==[] or second==():
    4509              return {1:0}
    4510          items = [self.expand(item, first.type) for item in second]
    4511          return {self.expand(first) : {"$in" : items} }
    4512
    4513      #TODO verify full compatibilty with official SQL Like operator
    4514 -    def LIKE(self, first, second):
    4515          import re
    4516          return {self.expand(first) : {'$regex' : re.escape(self.expand(second, 'string')).replace('%','.*')}}
    4517
    4518      #TODO verify full compatibilty with official SQL Like operator
    4519 -    def STARTSWITH(self, first, second):
    4520          #TODO  Solve almost the same problem as with endswith
    4521          import re
    4522          return {self.expand(first) : {'$regex' : '^' + re.escape(self.expand(second, 'string'))}}
    4523
    4524      #TODO verify full compatibilty with official SQL Like operator
    4525 -    def ENDSWITH(self, first, second):
    4526          #escaping regex operators?
    4527          #TODO if searched for a name like zsa_corbitt and the function is endswith('a') then this is also returned. Aldo it end with a t
    4528          import re
    4529          return {self.expand(first) : {'$regex' : re.escape(self.expand(second, 'string')) + '$'}}
    4530
    4531      #TODO verify full compatibilty with official oracle contains operator
    4532 -    def CONTAINS(self, first, second):
    4533          #There is a technical difference, but mongodb doesn't support that, but the result will be the same
    4534          #TODO contains operators need to be transformed to Regex
    4535          return {self.expand(first) : {' $regex' : ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
    4536
    4537 -    def EQ(self,first,second):
    4538          result = {}
    4539          #if second is None:
    4540              #return '(%s == null)' % self.expand(first)
    4541          #return '(%s == %s)' % (self.expand(first),self.expand(second,first.type))
    4542          result[self.expand(first)] = self.expand(second)
    4543          return result
    4544
    4545 -    def NE(self, first, second=None):
    4546          print "in NE"
    4547          result = {}
    4548          result[self.expand(first)] = {'$ne': self.expand(second)}
    4549          return result
    4550
    4551 -    def LT(self,first,second=None):
    4552          if second is None:
    4553              raise RuntimeError, "Cannot compare %s < None" % first
    4554          print "in LT"
    4555          result = {}
    4556          result[self.expand(first)] = {'$lt': self.expand(second)}
    4557          return result
    4558
    4559 -    def LE(self,first,second=None):
    4560          if second is None:
    4561              raise RuntimeError, "Cannot compare %s <= None" % first
    4562          print "in LE"
    4563          result = {}
    4564          result[self.expand(first)] = {'$lte': self.expand(second)}
    4565          return result
    4566
    4567 -    def GT(self,first,second):
    4568          print "in GT"
    4569          #import pymongo.objectid
    4570          result = {}
    4571          #if expanded_first == '_id':
    4572              #if expanded_second != 0 and not isinstance(second,pymongo.objectid.ObjectId):
    4573                  #raise SyntaxError, 'second argument must be of type bson.objectid.ObjectId'
    4574              #elif expanded_second == 0:
    4575                  #expanded_second = pymongo.objectid.ObjectId('000000000000000000000000')
    4576          result[self.expand(first)] = {'$gt': self.expand(second)}
    4577          return result
    4578
    4579 -    def GE(self,first,second=None):
    4580          if second is None:
    4581              raise RuntimeError, "Cannot compare %s >= None" % first
    4582          print "in GE"
    4583          result = {}
    4584          result[self.expand(first)] = {'$gte': self.expand(second)}
    4585          return result
    4586
    4587      #TODO javascript has math
    4588 -    def ADD(self, first, second):
    4589          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4590          return '%s + %s' % (self.expand(first), self.expand(second, first.type))
    4591
    4592      #TODO javascript has math
    4593 -    def SUB(self, first, second):
    4594          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4595          return '(%s - %s)' % (self.expand(first), self.expand(second, first.type))
    4596
    4597      #TODO javascript has math
    4598 -    def MUL(self, first, second):
    4599          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4600          return '(%s * %s)' % (self.expand(first), self.expand(second, first.type))
    4601          #TODO javascript has math
    4602
    4603 -    def DIV(self, first, second):
    4604          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4605          return '(%s / %s)' % (self.expand(first), self.expand(second, first.type))
    4606      #TODO javascript has math
    4607 -    def MOD(self, first, second):
    4608          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4609          return '(%s %% %s)' % (self.expand(first), self.expand(second, first.type))
    4610
    4611      #TODO javascript can do this
    4612 -    def AS(self, first, second):
    4613          raise NotImplementedError, "This must yet be replaced with javescript in order to accomplish this. Sorry"
    4614          return '%s AS %s' % (self.expand(first), second)
    4615
    4616      #We could implement an option that simulates a full featured SQL database. But I think the option should be set explicit or implemented as another library.
    4617 -    def ON(self, first, second):
    4618          raise NotImplementedError, "This is not possible in NoSQL, but can be simulated with a wrapper."
    4619          return '%s ON %s' % (self.expand(first), self.expand(second))
    4620
    4621      #TODO is this used in mongodb?
    4622 -    def COMMA(self, first, second):
    4623          return '%s, %s' % (self.expand(first), self.expand(second))
    4624
    4625
    4626
    4627 -class IMAPAdapter(NoSQLAdapter):
    4628      """ IMAP server adapter
    4629
    4630        This class is intended as an interface with
    4631      email IMAP servers to perform simple queries in the
    4632      web2py DAL query syntax, so email read, search and
    4633      other related IMAP mail services (as those implemented
    4634      by brands like Google(r), and Yahoo!(r)
    4635      can be managed from web2py applications.
    4636
    4637      The code uses examples by Yuji Tomita on this post:
    4638      http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
    4639      and is based in docs for Python imaplib, python email
    4640      and email IETF's (i.e. RFC2060 and RFC3501)
    4641
    4642      This adapter was tested with a small set of operations with Gmail(r). Other
    4643      services requests could raise command syntax and response data issues.
    4644
    4645      It creates its table and field names "statically",
    4646      meaning that the developer should leave the table and field
    4647      definitions to the DAL instance by calling the adapter's
    4648      .define_tables() method. The tables are defined with the
    4649      IMAP server mailbox list information.
    4650
    4651      Here is a list of supported fields:
    4652
    4653      Field       Type            Description
    4654      ################################################################
    4655      uid         string
    4656      answered    boolean        Flag
    4657      created     date
    4658      content     list:string    A list of text or html parts
    4659      to          string
    4660      cc          string
    4661      bcc         string
    4662      size        integer        the amount of octets of the message*
    4663      deleted     boolean        Flag
    4664      draft       boolean        Flag
    4665      flagged     boolean        Flag
    4666      sender      string
    4667      recent      boolean        Flag
    4668      seen        boolean        Flag
    4669      subject     string
    4670      mime        string         The mime header declaration
    4671      email       string         The complete RFC822 message**
    4672      attachments list:string    Each non text decoded part as string
    4673
    4674      *At the application side it is measured as the length of the RFC822
    4675      message string
    4676
    4677      WARNING: As row id's are mapped to email sequence numbers,
    4678      make sure your imap client web2py app does not delete messages
    4679      during select or update actions, to prevent
    4680      updating or deleting different messages.
    4681      Sequence numbers change whenever the mailbox is updated.
    4682      To avoid this sequence numbers issues, it is recommended the use
    4683      of uid fields in query references (although the update and delete
    4684      in separate actions rule still applies).
    4685
    4686      # This is the code recommended to start imap support
    4687      # at the app's model:
    4688
    4689      imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
    4690      imapdb.define_tables()
    4691
    4692      Here is an (incomplete) list of possible imap commands:
    4693
    4694      # Count today's unseen messages
    4695      # smaller than 6000 octets from the
    4696      # inbox mailbox
    4697
    4698      q = imapdb.INBOX.seen == False
    4699      q &= imapdb.INBOX.created == datetime.date.today()
    4700      q &= imapdb.INBOX.size < 6000
    4701      unread = imapdb(q).count()
    4702
    4703      # Fetch last query messages
    4704      rows = imapdb(q).select()
    4705
    4706      # it is also possible to filter query select results with limitby and
    4707      # sequences of mailbox fields
    4708
    4709      set.select(<fields sequence>, limitby=(<int>, <int>))
    4710
    4711      # Mark last query messages as seen
    4712      messages = [row.uid for row in rows]
    4713      seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)
    4714
    4715      # Delete messages in the imap database that have mails from mr. Gumby
    4716
    4717      deleted = 0
    4718      for mailbox in imapdb.tables
    4719          deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()
    4720
    4721      # It is possible also to mark messages for deletion instead of ereasing them
    4722      # directly with set.update(deleted=True)
    4723
    4724      """
    4725
    4726      types = {
    4727                  'string': str,
    4728                  'text': str,
    4729                  'date': datetime.date,
    4730                  'datetime': datetime.datetime,
    4731                  'id': long,
    4732                  'boolean': bool,
    4733                  'integer': int,
    4734                  'blob': str,
    4735                  'list:string': str,
    4736          }
    4737
    4738      dbengine = 'imap'
    4739      driver = globals().get('imaplib',None)
    4740
    4741 -    def __init__(self,
    4742                   db,
    4743                   uri,
    4744                   pool_size=0,
    4745                   folder=None,
    4746                   db_codec ='UTF-8',
    4747                   credential_decoder=lambda x:x,
    4748                   driver_args={},
    4749                   adapter_args={}):
    4750
    4751          # db uri: user@example.com:password@imap.server.com:123
    4752          # TODO: max size adapter argument for preventing large mail transfers
    4753
    4754          uri = uri.split("://")[1]
    4755          self.db = db
    4756          self.uri = uri
    4757          self.pool_size=pool_size
    4758          self.folder = folder
    4759          self.db_codec = db_codec
    4760          self.credential_decoder = credential_decoder
    4761          self.driver_args = driver_args
    4762          self.adapter_args = adapter_args
    4763          self.mailbox_size = None
    4764          self.charset = sys.getfilesystemencoding()
    4765          # imap class
    4766          self.imap4 = None
    4767
    4768          """ MESSAGE is an identifier for sequence number"""
    4769
    4770          self.flags = ['\\Deleted', '\\Draft', '\\Flagged',
    4771                        '\\Recent', '\\Seen', '\\Answered']
    4772          self.search_fields = {
    4773              'id': 'MESSAGE', 'created': 'DATE',
    4774              'uid': 'UID', 'sender': 'FROM',
    4775              'to': 'TO', 'cc': 'CC',
    4776              'bcc': 'BCC', 'content': 'TEXT',
    4777              'size': 'SIZE', 'deleted': '\\Deleted',
    4778              'draft': '\\Draft', 'flagged': '\\Flagged',
    4779              'recent': '\\Recent', 'seen': '\\Seen',
    4780              'subject': 'SUBJECT', 'answered': '\\Answered',
    4781              'mime': None, 'email': None,
    4782              'attachments': None
    4783              }
    4784
    4785          db['_lastsql'] = ''
    4786
    4787          m = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@/]+)(\:(?P<port>[0-9]+))?$').match(uri)
    4788          user = m.group('user')
    4789          password = m.group('password')
    4790          host = m.group('host')
    4791          port = int(m.group('port'))
    4792          over_ssl = False
    4793          if port==993:
    4794              over_ssl = True
    4795
    4796          driver_args.update(dict(host=host,port=port, password=password, user=user))
    4797          def connect(driver_args=driver_args):
    4798              # it is assumed sucessful authentication alLways
    4799              # TODO: support direct connection and login tests
    4800              if over_ssl:
    4801                  self.imap4 = self.driver.IMAP4_SSL
    4802              else:
    4803                  self.imap4 = self.driver.IMAP4
    4804              connection = self.imap4(driver_args["host"], driver_args["port"])
    4805              data = connection.login(driver_args["user"], driver_args["password"])
    4806              # print "Connected to remote server"
    4807              # print data
    4808              # static mailbox list
    4809              connection.mailbox_names = None
    4810
    4811              # dummy cursor function
    4812              connection.cursor = lambda : True
    4813
    4814              return connection
    4815
    4816          self.pool_connection(connect)
    4817          self.db.define_tables = self.define_tables
    4818
    4819 -    def pool_connection(self, f, cursor=True):
    4820          """
    4821          IMAP4 Pool connection method
    4822
    4823          imap connection lacks of self cursor command.
    4824          A custom command should be provided as a replacement
    4825          for connection pooling to prevent uncaught remote session
    4826          closing
    4827
    4828          """
    4829          # print "Pool Connection"
    4830          if not self.pool_size:
    4831              self.connection = f()
    4832              self.cursor = cursor and self.connection.cursor()
    4833          else:
    4834              uri = self.uri
    4835              # print "uri", self.uri
    4836              while True:
    4837                  sql_locker.acquire()
    4838                  if not uri in ConnectionPool.pools:
    4839                      ConnectionPool.pools[uri] = []
    4840                  if ConnectionPool.pools[uri]:
    4841                      self.connection = ConnectionPool.pools[uri].pop()
    4842                      sql_locker.release()
    4843                      self.cursor = cursor and self.connection.cursor()
    4844                      # print "self.cursor", self.cursor
    4845                      if self.cursor and self.check_active_connection:
    4846                          try:
    4847                              # check if connection is alive or close it
    4848                              result, data = self.connection.list()
    4849                              # print "Checked connection"
    4850                              # print result, data
    4851                              # self.execute('SELECT 1;')
    4852                          except:
    4853                              # Possible connection reset error
    4854                              # TODO: read exception class
    4855                              # print "Re-connecting to IMAP server"
    4856                              self.connection = f()
    4857                      break
    4858                  else:
    4859                      sql_locker.release()
    4860                      self.connection = f()
    4861                      self.cursor = cursor and self.connection.cursor()
    4862                      break
    4863
    4864          if not hasattr(thread,'instances'):
    4865              thread.instances = []
    4866          thread.instances.append(self)
    4867
    4868 -    def get_last_message(self, tablename):
    4869          last_message = None
    4870          # request mailbox list to the server
    4871          # if needed
    4872          if not isinstance(self.connection.mailbox_names, dict):
    4873              self.get_mailboxes()
    4874          try:
    4875              result = self.connection.select(self.connection.mailbox_names[tablename])
    4876              last_message = int(result[1][0])
    4877          except (IndexError, ValueError, TypeError, KeyError), e:
    4878              logger.debug("Error retrieving the last mailbox sequence number. %s" % str(e))
    4879          return last_message
    4880
    4881 -    def get_uid_bounds(self, tablename):
    4882          if not isinstance(self.connection.mailbox_names, dict):
    4883              self.get_mailboxes()
    4884          # fetch first and last messages
    4885          # return (first, last) messages uid's
    4886          last_message = self.get_last_message(tablename)
    4887          result, data = self.connection.uid("search", None, "(ALL)")
    4888          uid_list = data[0].strip().split()
    4889          if len(uid_list) <= 0:
    4890              return None
    4891          else:
    4892              return (uid_list[0], uid_list[-1])
    4893
    4894 -    def convert_date(self, date, add=None):
    4895          if add is None:
    4896              add = datetime.timedelta()
    4897          """ Convert a date object to a string
    4898          with d-Mon-Y style for IMAP or the inverse
    4899          case
    4900
    4901          add <timedelta> adds to the date object
    4902          """
    4903          months = [None, "Jan","Feb","Mar","Apr","May","Jun",
    4904                    "Jul", "Aug","Sep","Oct","Nov","Dec"]
    4905          if isinstance(date, basestring):
    4906              # Prevent unexpected date response format
    4907              try:
    4908                  dayname, datestring = date.split(",")
    4909              except (ValueError):
    4910                  logger.debug("Could not parse date text: %s" % date)
    4911                  return None
    4912              date_list = datestring.strip().split()
    4913              year = int(date_list[2])
    4914              month = months.index(date_list[1])
    4915              day = int(date_list[0])
    4916              hms = [int(value) for value in date_list[3].split(":")]
    4917              return datetime.datetime(year, month, day,
    4918                                       hms[0], hms[1], hms[2]) + add
    4919          elif isinstance(date, (datetime.datetime, datetime.date)):
    4920              return (date + add).strftime("%d-%b-%Y")
    4921
    4922          else:
    4923              return None
    4924
    4925 -    def encode_text(self, text, charset, errors="replace"):
    4926          """ convert text for mail to unicode"""
    4927          if text is None:
    4928              text = ""
    4929          else:
    4930              if isinstance(text, str):
    4931                  if charset is not None:
    4932                      text = unicode(text, charset, errors)
    4933                  else:
    4934                      text = unicode(text, "utf-8", errors)
    4935              else:
    4936                  raise Exception("Unsupported mail text type %s" % type(text))
    4937          return text.encode("utf-8")
    4938
    4939 -    def get_charset(self, message):
    4940          charset = message.get_content_charset()
    4941          return charset
    4942
    4943 -    def reset_mailboxes(self):
    4944          self.connection.mailbox_names = None
    4945          self.get_mailboxes()
    4946
    4947 -    def get_mailboxes(self):
    4948          """ Query the mail database for mailbox names """
    4949          mailboxes_list = self.connection.list()
    4950          self.connection.mailbox_names = dict()
    4951          mailboxes = list()
    4952          for item in mailboxes_list[1]:
    4953              item = item.strip()
    4954              if not "NOSELECT" in item.upper():
    4955                  sub_items = item.split("\"")
    4956                  sub_items = [sub_item for sub_item in sub_items if len(sub_item.strip()) > 0]
    4957                  mailbox = sub_items[len(sub_items) - 1]
    4958                  # remove unwanted characters and store original names
    4959                  mailbox_name = mailbox.replace("[", "").replace("]", "").replace("/", "_")
    4960                  mailboxes.append(mailbox_name)
    4961                  self.connection.mailbox_names[mailbox_name] = mailbox
    4962          # print "Mailboxes query", mailboxes
    4963          return mailboxes
    4964
    4965 -    def get_query_mailbox(self, query):
    4966          nofield = True
    4967          tablename = None
    4968          attr = query
    4969          while nofield:
    4970              if hasattr(attr, "first"):
    4971                  attr = attr.first
    4972                  if isinstance(attr, Field):
    4973                      return attr.tablename
    4974                  elif isinstance(attr, Query):
    4975                      pass
    4976                  else:
    4977                      return None
    4978              else:
    4979                  return None
    4980          return tablename
    4981
    4982 -    def is_flag(self, flag):
    4983          if self.search_fields.get(flag, None) in self.flags:
    4984              return True
    4985          else:
    4986              return False
    4987
    4988 -    def define_tables(self):
    4989          """
    4990          Auto create common IMAP fileds
    4991
    4992          This function creates fields definitions "statically"
    4993          meaning that custom fields as in other adapters should
    4994          not be supported and definitions handled on a service/mode
    4995          basis (local syntax for Gmail(r), Ymail(r)
    4996          """
    4997          if not isinstance(self.connection.mailbox_names, dict):
    4998              self.get_mailboxes()
    4999          mailboxes = self.connection.mailbox_names.keys()
    5000          for mailbox_name in mailboxes:
    5001              self.db.define_table("%s" % mailbox_name,
    5002                              Field("uid", "string", writable=False),
    5003                              Field("answered", "boolean"),
    5004                              Field("created", "datetime", writable=False),
    5005                              Field("content", "list:string", writable=False),
    5006                              Field("to", "string", writable=False),
    5007                              Field("cc", "string", writable=False),
    5008                              Field("bcc", "string", writable=False),
    5009                              Field("size", "integer", writable=False),
    5010                              Field("deleted", "boolean"),
    5011                              Field("draft", "boolean"),
    5012                              Field("flagged", "boolean"),
    5013                              Field("sender", "string", writable=False),
    5014                              Field("recent", "boolean", writable=False),
    5015                              Field("seen", "boolean"),
    5016                              Field("subject", "string", writable=False),
    5017                              Field("mime", "string", writable=False),
    5018                              Field("email", "string", writable=False, readable=False),
    5019                              Field("attachments", "list:string", writable=False, readable=False),
    5020                              )
    5021
    5022 -    def create_table(self, *args, **kwargs):
    5023          # not implemented
    5024          logger.debug("Create table feature is not implemented for %s" % type(self))
    5025
    5026 -    def _select(self,query,fields,attributes):
    5027          """  Search and Fetch records and return web2py
    5028          rows
    5029          """
    5030
    5031          if use_common_filters(query):
    5032              query = self.common_filter(query, [self.get_query_mailbox(query),])
    5033
    5034          # move this statement elsewhere (upper-level)
    5035          import email
    5036          import email.header
    5037          decode_header = email.header.decode_header
    5038          # get records from imap server with search + fetch
    5039          # convert results to a dictionary
    5040          tablename = None
    5041          fetch_results = list()
    5042          if isinstance(query, (Expression, Query)):
    5043              tablename = self.get_table(query)
    5044              mailbox = self.connection.mailbox_names.get(tablename, None)
    5045              if isinstance(query, Expression):
    5046                  pass
    5047              elif isinstance(query, Query):
    5048                  if mailbox is not None:
    5049                      # select with readonly
    5050                      selected = self.connection.select(mailbox, True)
    5051                      self.mailbox_size = int(selected[1][0])
    5052                      search_query = "(%s)" % str(query).strip()
    5053                      # print "Query", query
    5054                      # print "Search query", search_query
    5055                      search_result = self.connection.uid("search", None, search_query)
    5056                      # print "Search result", search_result
    5057                      # print search_result
    5058                      # Normal IMAP response OK is assumed (change this)
    5059                      if search_result[0] == "OK":
    5060                          # For "light" remote server responses just get the first
    5061                          # ten records (change for non-experimental implementation)
    5062                          # However, light responses are not guaranteed with this
    5063                          # approach, just fewer messages.
    5064                          # TODO: change limitby single to 2-tuple argument
    5065                          limitby = attributes.get('limitby', None)
    5066                          messages_set = search_result[1][0].split()
    5067                          # descending order
    5068                          messages_set.reverse()
    5069                          if limitby is not None:
    5070                              # TODO: asc/desc attributes
    5071                              messages_set = messages_set[int(limitby[0]):int(limitby[1])]
    5072                          # Partial fetches are not used since the email
    5073                          # library does not seem to support it (it converts
    5074                          # partial messages to mangled message instances)
    5075                          imap_fields = "(RFC822)"
    5076                          if len(messages_set) > 0:
    5077                              # create fetch results object list
    5078                              # fetch each remote message and store it in memmory
    5079                              # (change to multi-fetch command syntax for faster
    5080                              # transactions)
    5081                              for uid in messages_set:
    5082                                  # fetch the RFC822 message body
    5083                                  typ, data = self.connection.uid("fetch", uid, imap_fields)
    5084                                  if typ == "OK":
    5085                                      fr = {"message": int(data[0][0].split()[0]),
    5086                                          "uid": int(uid),
    5087                                          "email": email.message_from_string(data[0][1]),
    5088                                          "raw_message": data[0][1]
    5089                                          }
    5090                                      fr["multipart"] = fr["email"].is_multipart()
    5091                                      # fetch flags for the message
    5092                                      ftyp, fdata = self.connection.uid("fetch", uid, "(FLAGS)")
    5093                                      if ftyp == "OK":
    5094                                          # print "Raw flags", fdata
    5095                                          fr["flags"] = self.driver.ParseFlags(fdata[0])
    5096                                          # print "Flags", fr["flags"]
    5097                                          fetch_results.append(fr)
    5098                                      else:
    5099                                          # error retrieving the flags for this message
    5100                                          pass
    5101                                  else:
    5102                                      # error retrieving the message body
    5103                                      pass
    5104
    5105          elif isinstance(query, basestring):
    5106              # not implemented
    5107              pass
    5108          else:
    5109              pass
    5110
    5111          imapqry_dict = {}
    5112          imapfields_dict = {}
    5113
    5114          if len(fields) == 1 and isinstance(fields[0], SQLALL):
    5115              allfields = True
    5116          elif len(fields) == 0:
    5117              allfields = True
    5118          else:
    5119              allfields = False
    5120          if allfields:
    5121              fieldnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
    5122          else:
    5123              fieldnames = ["%s.%s" % (tablename, field.name) for field in fields]
    5124
    5125          for k in fieldnames:
    5126              imapfields_dict[k] = k
    5127
    5128          imapqry_list = list()
    5129          imapqry_array = list()
    5130          for fr in fetch_results:
    5131              attachments = []
    5132              content = []
    5133              size = 0
    5134              n = int(fr["message"])
    5135              item_dict = dict()
    5136              message = fr["email"]
    5137              uid = fr["uid"]
    5138              charset = self.get_charset(message)
    5139              flags = fr["flags"]
    5140              raw_message = fr["raw_message"]
    5141              # Return messages data mapping static fields
    5142              # and fetched results. Mapping should be made
    5143              # outside the select function (with auxiliary
    5144              # instance methods)
    5145
    5146              # pending: search flags states trough the email message
    5147              # instances for correct output
    5148
    5149              if "%s.id" % tablename in fieldnames:
    5150                  item_dict["%s.id" % tablename] = n
    5151              if "%s.created" % tablename in fieldnames:
    5152                  item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
    5153              if "%s.uid" % tablename in fieldnames:
    5154                  item_dict["%s.uid" % tablename] = uid
    5155              if "%s.sender" % tablename in fieldnames:
    5156                  # If there is no encoding found in the message header
    5157                  # force utf-8 replacing characters (change this to
    5158                  # module's defaults). Applies to .sender, .to, .cc and .bcc fields
    5159                  #############################################################################
    5160                  # TODO: External function to manage encoding and decoding of message strings
    5161                  #############################################################################
    5162                  item_dict["%s.sender" % tablename] = self.encode_text(message["From"], charset)
    5163              if "%s.to" % tablename in fieldnames:
    5164                  item_dict["%s.to" % tablename] = self.encode_text(message["To"], charset)
    5165              if "%s.cc" % tablename in fieldnames:
    5166                  if "Cc" in message.keys():
    5167                      # print "cc field found"
    5168                      item_dict["%s.cc" % tablename] = self.encode_text(message["Cc"], charset)
    5169                  else:
    5170                      item_dict["%s.cc" % tablename] = ""
    5171              if "%s.bcc" % tablename in fieldnames:
    5172                  if "Bcc" in message.keys():
    5173                      # print "bcc field found"
    5174                      item_dict["%s.bcc" % tablename] = self.encode_text(message["Bcc"], charset)
    5175                  else:
    5176                      item_dict["%s.bcc" % tablename] = ""
    5177              if "%s.deleted" % tablename in fieldnames:
    5178                  item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
    5179              if "%s.draft" % tablename in fieldnames:
    5180                  item_dict["%s.draft" % tablename] = "\\Draft" in flags
    5181              if "%s.flagged" % tablename in fieldnames:
    5182                  item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
    5183              if "%s.recent" % tablename in fieldnames:
    5184                  item_dict["%s.recent" % tablename] = "\\Recent" in flags
    5185              if "%s.seen" % tablename in fieldnames:
    5186                  item_dict["%s.seen" % tablename] = "\\Seen" in flags
    5187              if "%s.subject" % tablename in fieldnames:
    5188                  subject = message["Subject"]
    5189                  decoded_subject = decode_header(subject)
    5190                  text = decoded_subject[0][0]
    5191                  encoding = decoded_subject[0][1]
    5192                  if encoding in (None, ""):
    5193                      encoding = charset
    5194                  item_dict["%s.subject" % tablename] = self.encode_text(text, encoding)
    5195              if "%s.answered" % tablename in fieldnames:
    5196                  item_dict["%s.answered" % tablename] = "\\Answered" in flags
    5197              if "%s.mime" % tablename in fieldnames:
    5198                  item_dict["%s.mime" % tablename] = message.get_content_type()
    5199              # Here goes the whole RFC822 body as an email instance
    5200              # for controller side custom processing
    5201              # The message is stored as a raw string
    5202              # >> email.message_from_string(raw string)
    5203              # returns a Message object for enhanced object processing
    5204              if "%s.email" % tablename in fieldnames:
    5205                  item_dict["%s.email" % tablename] = self.encode_text(raw_message, charset)
    5206              # Size measure as suggested in a Velocity Reviews post
    5207              # by Tim Williams: "how to get size of email attachment"
    5208              # Note: len() and server RFC822.SIZE reports doesn't match
    5209              # To retrieve the server size for representation would add a new
    5210              # fetch transaction to the process
    5211              for part in message.walk():
    5212                  if "%s.attachments" % tablename in fieldnames:
    5213                      if not "text" in part.get_content_maintype():
    5214                          attachments.append(part.get_payload(decode=True))
    5215                  if "%s.content" % tablename in fieldnames:
    5216                      if "text" in part.get_content_maintype():
    5217                          payload = self.encode_text(part.get_payload(decode=True), charset)
    5218                          content.append(payload)
    5219                  if "%s.size" % tablename in fieldnames:
    5220                      if part is not None:
    5221                          size += len(str(part))
    5222
    5223              item_dict["%s.content" % tablename] = bar_encode(content)
    5224              item_dict["%s.attachments" % tablename] = bar_encode(attachments)
    5225              item_dict["%s.size" % tablename] = size
    5226
    5227              imapqry_list.append(item_dict)
    5228
    5229          # extra object mapping for the sake of rows object
    5230          # creation (sends an array or lists)
    5231          for item_dict in imapqry_list:
    5232              imapqry_array_item = list()
    5233              for fieldname in fieldnames:
    5234                  imapqry_array_item.append(item_dict[fieldname])
    5235              imapqry_array.append(imapqry_array_item)
    5236
    5237          return tablename, imapqry_array, fieldnames
    5238
    5239 -    def select(self,query,fields,attributes):
    5240          tablename, imapqry_array , fieldnames = self._select(query,fields,attributes)
    5241          # parse result and return a rows object
    5242          colnames = fieldnames
    5243          result = self.parse(imapqry_array, fields, colnames)
    5244          return result
    5245
    5246 -    def update(self, tablename, query, fields):
    5247          # print "_update"
    5248
    5249          if use_common_filters(query):
    5250              query = self.common_filter(query, [tablename,])
    5251
    5252          mark = []
    5253          unmark = []
    5254          rowcount = 0
    5255          query = str(query)
    5256          if query:
    5257              for item in fields:
    5258                  field = item[0]
    5259                  name = field.name
    5260                  value = item[1]
    5261                  if self.is_flag(name):
    5262                      flag = self.search_fields[name]
    5263                      if (value is not None) and (flag != "\\Recent"):
    5264                          if value:
    5265                              mark.append(flag)
    5266                          else:
    5267                              unmark.append(flag)
    5268
    5269              # print "Selecting mailbox ..."
    5270              result, data = self.connection.select(self.connection.mailbox_names[tablename])
    5271              # print "Retrieving sequence numbers remotely"
    5272              string_query = "(%s)" % query
    5273              # print "string query", string_query
    5274              result, data = self.connection.search(None, string_query)
    5275              store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
    5276              # print "Storing values..."
    5277              # change marked flags
    5278              for number in store_list:
    5279                  result = None
    5280                  if len(mark) > 0:
    5281                      # print "Marking flags ..."
    5282                      result, data = self.connection.store(number, "+FLAGS", "(%s)" % " ".join(mark))
    5283                  if len(unmark) > 0:
    5284                      # print "Unmarking flags ..."
    5285                      result, data = self.connection.store(number, "-FLAGS", "(%s)" % " ".join(unmark))
    5286                  if result == "OK":
    5287                      rowcount += 1
    5288          return rowcount
    5289
    5290 -    def count(self,query,distinct=None):
    5291          counter = 0
    5292          tablename = self.get_query_mailbox(query)
    5293          if query and tablename is not None:
    5294              if use_common_filters(query):
    5295                  query = self.common_filter(query, [tablename,])
    5296              # print "Selecting mailbox ..."
    5297              result, data = self.connection.select(self.connection.mailbox_names[tablename])
    5298              # print "Retrieving sequence numbers remotely"
    5299              string_query = "(%s)" % query
    5300              result, data = self.connection.search(None, string_query)
    5301              store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
    5302              counter = len(store_list)
    5303          return counter
    5304
    5305 -    def delete(self, tablename, query):
    5306          counter = 0
    5307          if query:
    5308              # print "Selecting mailbox ..."
    5309              if use_common_filters(query):
    5310                  query = self.common_filter(query, [tablename,])
    5311              result, data = self.connection.select(self.connection.mailbox_names[tablename])
    5312              # print "Retrieving sequence numbers remotely"
    5313              string_query = "(%s)" % query
    5314              result, data = self.connection.search(None, string_query)
    5315              store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
    5316              for number in store_list:
    5317                  result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
    5318                  # print "Deleting message", result, data
    5319                  if result == "OK":
    5320                      counter += 1
    5321              if counter > 0:
    5322                  # print "Ereasing permanently"
    5323                  result, data = self.connection.expunge()
    5324          return counter
    5325
    5326 -    def BELONGS(self, first, second):
    5327          result = None
    5328          name = self.search_fields[first.name]
    5329          if name == "MESSAGE":
    5330              values = [str(val) for val in second if str(val).isdigit()]
    5331              result = "%s" % ",".join(values).strip()
    5332
    5333          elif name == "UID":
    5334              values = [str(val) for val in second if str(val).isdigit()]
    5335              result = "UID %s" % ",".join(values).strip()
    5336
    5337          else:
    5338              raise Exception("Operation not supported")
    5339          # result = "(%s %s)" % (self.expand(first), self.expand(second))
    5340          return result
    5341
    5342 -    def CONTAINS(self, first, second):
    5343          result = None
    5344          name = self.search_fields[first.name]
    5345
    5346          if name in ("FROM", "TO", "SUBJECT", "TEXT"):
    5347              result = "%s \"%s\"" % (name, self.expand(second))
    5348          else:
    5349              if first.name in ("cc", "bcc"):
    5350                  result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
    5351              elif first.name == "mime":
    5352                  result = "HEADER Content-Type \"%s\"" % self.expand(second)
    5353              else:
    5354                  raise Exception("Operation not supported")
    5355          return result
    5356
    5357 -    def GT(self, first, second):
    5358          result = None
    5359          name = self.search_fields[first.name]
    5360          if name == "MESSAGE":
    5361              last_message = self.get_last_message(first.tablename)
    5362              result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
    5363          elif name == "UID":
    5364              # GT and LT may not return
    5365              # expected sets depending on
    5366              # the uid format implemented
    5367              try:
    5368                  pedestal, threshold = self.get_uid_bounds(first.tablename)
    5369              except TypeError, e:
    5370                  logger.debug("Error requesting uid bounds: %s", str(e))
    5371                  return ""
    5372              try:
    5373                  lower_limit = int(self.expand(second)) + 1
    5374              except (ValueError, TypeError), e:
    5375                  raise Exception("Operation not supported (non integer UID)")
    5376              result = "UID %s:%s" % (lower_limit, threshold)
    5377          elif name == "DATE":
    5378              result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
    5379          elif name == "SIZE":
    5380              result = "LARGER %s" % self.expand(second)
    5381          else:
    5382              raise Exception("Operation not supported")
    5383          return result
    5384
    5385 -    def GE(self, first, second):
    5386          result = None
    5387          name = self.search_fields[first.name]
    5388          if name == "MESSAGE":
    5389              last_message = self.get_last_message(first.tablename)
    5390              result = "%s:%s" % (self.expand(second), last_message)
    5391          elif name == "UID":
    5392              # GT and LT may not return
    5393              # expected sets depending on
    5394              # the uid format implemented
    5395              try:
    5396                  pedestal, threshold = self.get_uid_bounds(first.tablename)
    5397              except TypeError, e:
    5398                  logger.debug("Error requesting uid bounds: %s", str(e))
    5399                  return ""
    5400              lower_limit = self.expand(second)
    5401              result = "UID %s:%s" % (lower_limit, threshold)
    5402          elif name == "DATE":
    5403              result = "SINCE %s" % self.convert_date(second)
    5404          else:
    5405              raise Exception("Operation not supported")
    5406          return result
    5407
    5408 -    def LT(self, first, second):
    5409          result = None
    5410          name = self.search_fields[first.name]
    5411          if name == "MESSAGE":
    5412              result = "%s:%s" % (1, int(self.expand(second)) - 1)
    5413          elif name == "UID":
    5414              try:
    5415                  pedestal, threshold = self.get_uid_bounds(first.tablename)
    5416              except TypeError, e:
    5417                  logger.debug("Error requesting uid bounds: %s", str(e))
    5418                  return ""
    5419              try:
    5420                  upper_limit = int(self.expand(second)) - 1
    5421              except (ValueError, TypeError), e:
    5422                  raise Exception("Operation not supported (non integer UID)")
    5423              result = "UID %s:%s" % (pedestal, upper_limit)
    5424          elif name == "DATE":
    5425              result = "BEFORE %s" % self.convert_date(second)
    5426          elif name == "SIZE":
    5427              result = "SMALLER %s" % self.expand(second)
    5428          else:
    5429              raise Exception("Operation not supported")
    5430          return result
    5431
    5432 -    def LE(self, first, second):
    5433          result = None
    5434          name = self.search_fields[first.name]
    5435          if name == "MESSAGE":
    5436              result = "%s:%s" % (1, self.expand(second))
    5437          elif name == "UID":
    5438              try:
    5439                  pedestal, threshold = self.get_uid_bounds(first.tablename)
    5440              except TypeError, e:
    5441                  logger.debug("Error requesting uid bounds: %s", str(e))
    5442                  return ""
    5443              upper_limit = int(self.expand(second))
    5444              result = "UID %s:%s" % (pedestal, upper_limit)
    5445          elif name == "DATE":
    5446              result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
    5447          else:
    5448              raise Exception("Operation not supported")
    5449          return result
    5450
    5451 -    def NE(self, first, second):
    5452          result = self.NOT(self.EQ(first, second))
    5453          result =  result.replace("NOT NOT", "").strip()
    5454          return result
    5455
    5456 -    def EQ(self,first,second):
    5457          name = self.search_fields[first.name]
    5458          result = None
    5459          if name is not None:
    5460              if name == "MESSAGE":
    5461                  # query by message sequence number
    5462                  result = "%s" % self.expand(second)
    5463              elif name == "UID":
    5464                  result = "UID %s" % self.expand(second)
    5465              elif name == "DATE":
    5466                  result = "ON %s" % self.convert_date(second)
    5467
    5468              elif name in self.flags:
    5469                  if second:
    5470                      result = "%s" % (name.upper()[1:])
    5471                  else:
    5472                      result = "NOT %s" % (name.upper()[1:])
    5473              else:
    5474                  raise Exception("Operation not supported")
    5475          else:
    5476              raise Exception("Operation not supported")
    5477          return result
    5478
    5479 -    def AND(self, first, second):
    5480          result = "%s %s" % (self.expand(first), self.expand(second))
    5481          return result
    5482
    5483 -    def OR(self, first, second):
    5484          result = "OR %s %s" % (self.expand(first), self.expand(second))
    5485          return "%s" % result.replace("OR OR", "OR")
    5486
    5487 -    def NOT(self, first):
    5488          result = "NOT %s" % self.expand(first)
    5489          return result
    5490
    5491  ########################################################################
    5492  # end of adapters
    5493  ########################################################################
    5494
    5495  ADAPTERS = {
    5496      'sqlite': SQLiteAdapter,
    5497      'sqlite:memory': SQLiteAdapter,
    5498      'mysql': MySQLAdapter,
    5499      'postgres': PostgreSQLAdapter,
    5500      'postgres:psycopg2': PostgreSQLAdapter,
    5501      'postgres:pg8000': PostgreSQLAdapter,
    5502      'oracle': OracleAdapter,
    5503      'mssql': MSSQLAdapter,
    5504      'mssql2': MSSQL2Adapter,
    5505      'db2': DB2Adapter,
    5506      'teradata': TeradataAdapter,
    5507      'informix': InformixAdapter,
    5508      'firebird': FireBirdAdapter,
    5509      'firebird_embedded': FireBirdAdapter,
    5510      'ingres': IngresAdapter,
    5511      'ingresu': IngresUnicodeAdapter,
    5512      'sapdb': SAPDBAdapter,
    5513      'cubrid': CubridAdapter,
    5514      'jdbc:sqlite': JDBCSQLiteAdapter,
    5515      'jdbc:sqlite:memory': JDBCSQLiteAdapter,
    5516      'jdbc:postgres': JDBCPostgreSQLAdapter,
    5517      'gae': GoogleDatastoreAdapter, # discouraged, for backward compatibility
    5518      'google:datastore': GoogleDatastoreAdapter,
    5519      'google:sql': GoogleSQLAdapter,
    5520      'couchdb': CouchDBAdapter,
    5521      'mongodb': MongoDBAdapter,
    5522      'imap': IMAPAdapter
    5523  }
    5524
    5525
    5526 -def sqlhtml_validators(field):
    5527      """
    5528      Field type validation, using web2py's validators mechanism.
    5529
    5530      makes sure the content of a field is in line with the declared
    5531      fieldtype
    5532      """
    5533      if not have_validators:
    5534          return []
    5535      field_type, field_length = field.type, field.length
    5536      if isinstance(field_type, SQLCustomType):
    5537          if hasattr(field_type, 'validator'):
    5538              return field_type.validator
    5539          else:
    5540              field_type = field_type.type
    5541      elif not isinstance(field_type,str):
    5542          return []
    5543      requires=[]
    5544      def ff(r,id):
    5545          row=r(id)
    5546          if not row:
    5547              return id
    5548          elif hasattr(r, '_format') and isinstance(r._format,str):
    5549              return r._format % row
    5550          elif hasattr(r, '_format') and callable(r._format):
    5551              return r._format(row)
    5552          else:
    5553              return id
    5554      if field_type == 'string':
    5555          requires.append(validators.IS_LENGTH(field_length))
    5556      elif field_type == 'text':
    5557          requires.append(validators.IS_LENGTH(field_length))
    5558      elif field_type == 'password':
    5559          requires.append(validators.IS_LENGTH(field_length))
    5560      elif field_type == 'double':
    5561          requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
    5562      elif field_type == 'integer':
    5563          requires.append(validators.IS_INT_IN_RANGE(-1e100, 1e100))
    5564      elif field_type.startswith('decimal'):
    5565          requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
    5566      elif field_type == 'date':
    5567          requires.append(validators.IS_DATE())
    5568      elif field_type == 'time':
    5569          requires.append(validators.IS_TIME())
    5570      elif field_type == 'datetime':
    5571          requires.append(validators.IS_DATETIME())
    5572      elif field.db and field_type.startswith('reference') and \
    5573              field_type.find('.') < 0 and \
    5574              field_type[10:] in field.db.tables:
    5575          referenced = field.db[field_type[10:]]
    5576          def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
    5577          field.represent = field.represent or repr_ref
    5578          if hasattr(referenced, '_format') and referenced._format:
    5579              requires = validators.IS_IN_DB(field.db,referenced._id,
    5580                                             referenced._format)
    5581              if field.unique:
    5582                  requires._and = validators.IS_NOT_IN_DB(field.db,field)
    5583              if field.tablename == field_type[10:]:
    5584                  return validators.IS_EMPTY_OR(requires)
    5585              return requires
    5586      elif field.db and field_type.startswith('list:reference') and \
    5587              field_type.find('.') < 0 and \
    5588              field_type[15:] in field.db.tables:
    5589          referenced = field.db[field_type[15:]]
    5590          def list_ref_repr(ids, row=None, r=referenced, f=ff):
    5591              if not ids:
    5592                  return None
    5593              refs = r._db(r._id.belongs(ids)).select(r._id)
    5594              return (refs and ', '.join(str(f(r,ref.id)) for ref in refs) or '')
    5595          field.represent = field.represent or list_ref_repr
    5596          if hasattr(referenced, '_format') and referenced._format:
    5597              requires = validators.IS_IN_DB(field.db,referenced._id,
    5598                                             referenced._format,multiple=True)
    5599          else:
    5600              requires = validators.IS_IN_DB(field.db,referenced._id,
    5601                                             multiple=True)
    5602          if field.unique:
    5603              requires._and = validators.IS_NOT_IN_DB(field.db,field)
    5604          return requires
    5605      elif field_type.startswith('list:'):
    5606          def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
    5607          field.represent = field.represent or repr_list
    5608      if field.unique:
    5609          requires.insert(0,validators.IS_NOT_IN_DB(field.db,field))
    5610      sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
    5611      if field.notnull and not field_type[:2] in sff:
    5612          requires.insert(0, validators.IS_NOT_EMPTY())
    5613      elif not field.notnull and field_type[:2] in sff and requires:
    5614          requires[-1] = validators.IS_EMPTY_OR(requires[-1])
    5615      return requires
    5616
    5617
    5618 -def bar_escape(item):
    5619      return str(item).replace('|', '||')
    5620
    5621 -def bar_encode(items):
    5622      return '|%s|' % '|'.join(bar_escape(item) for item in items if str(item).strip())
    5623
    5624 -def bar_decode_integer(value):
    5625      return [int(x) for x in value.split('|') if x.strip()]
    5626
    5627 -def bar_decode_string(value):
    5628      return [x.replace('||', '|') for x in string_unpack.split(value[1:-1]) if x.strip()]
    5629
    5630
    5631 -class Row(dict):
    5632
    5633      """
    5634      a dictionary that lets you do d['a'] as well as d.a
    5635      this is only used to store a Row
    5636      """
    5637
    5638 -    def __getitem__(self, key):
    5639          key=str(key)
    5640          m = regex_table_field.match(key)
    5641          if key in self.get('_extra',{}):
    5642              return self._extra[key]
    5643          elif m:
    5644              try:
    5645                  return dict.__getitem__(self, m.group(1))[m.group(2)]
    5646              except (KeyError,TypeError):
    5647                  key = m.group(2)
    5648          return dict.__getitem__(self, key)
    5649
    5650 -    def __call__(self,key):
    5651          return self.__getitem__(key)
    5652
    5653 -    def __setitem__(self, key, value):
    5654          dict.__setitem__(self, str(key), value)
    5655
    5656 -    def __getattr__(self, key):
    5657          return self[key]
    5658
    5659 -    def __setattr__(self, key, value):
    5660          self[key] = value
    5661
    5662 -    def __repr__(self):
    5663          return '<Row ' + dict.__repr__(self) + '>'
    5664
    5665 -    def __int__(self):
    5666          return dict.__getitem__(self,'id')
    5667
    5668 -    def __eq__(self,other):
    5669          try:
    5670              return self.as_dict() == other.as_dict()
    5671          except AttributeError:
    5672              return False
    5673
    5674 -    def __ne__(self,other):
    5675          return not (self == other)
    5676
    5677 -    def __copy__(self):
    5678          return Row(dict(self))
    5679
    5680 -    def as_dict(self,datetime_to_str=False):
    5681          SERIALIZABLE_TYPES = (str,unicode,int,long,float,bool,list)
    5682          d = dict(self)
    5683          for k in copy.copy(d.keys()):
    5684              v=d[k]
    5685              if d[k] is None:
    5686                  continue
    5687              elif isinstance(v,Row):
    5688                  d[k]=v.as_dict()
    5689              elif isinstance(v,Reference):
    5690                  d[k]=int(v)
    5691              elif isinstance(v,decimal.Decimal):
    5692                  d[k]=float(v)
    5693              elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
    5694                  if datetime_to_str:
    5695                      d[k] = v.isoformat().replace('T',' ')[:19]
    5696              elif not isinstance(v,SERIALIZABLE_TYPES):
    5697                  del d[k]
    5698          return d
    5699
    5700
    5701 -def Row_unpickler(data):
    5702      return Row(cPickle.loads(data))
    5703
    5704 -def Row_pickler(data):
    5705      return Row_unpickler, (cPickle.dumps(data.as_dict(datetime_to_str=False)),)
    5706
    5707  copy_reg.pickle(Row, Row_pickler, Row_unpickler)
    5708
    5709
    5710  ################################################################################
    5711  # Everything below should be independent on the specifics of the
    5712  # database and should for RDBMs and some NoSQL databases
    5713  ################################################################################
    5714
    5715 -class SQLCallableList(list):
    5716 -    def __call__(self):
    5717          return copy.copy(self)
    5718
    5719 -def smart_query(fields,text):
    5720      if not isinstance(fields,(list,tuple)):
    5721          fields = [fields]
    5722      new_fields = []
    5723      for field in fields:
    5724          if isinstance(field,Field):
    5725              new_fields.append(field)
    5726          elif isinstance(field,Table):
    5727              for ofield in field:
    5728                  new_fields.append(ofield)
    5729          else:
    5730              raise RuntimeError, "fields must be a list of fields"
    5731      fields = new_fields
    5732      field_map = {}
    5733      for field in fields:
    5734          n = field.name.lower()
    5735          if not n in field_map:
    5736              field_map[n] = field
    5737          n = str(field).lower()
    5738          if not n in field_map:
    5739              field_map[n] = field
    5740      re_constants = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
    5741      constants = {}
    5742      i = 0
    5743      while True:
    5744          m = re_constants.search(text)
    5745          if not m: break
    5746          text = text[:m.start()]+('#%i' % i)+text[m.end():]
    5747          constants[str(i)] = m.group()[1:-1]
    5748          i+=1
    5749      text = re.sub('\s+',' ',text).lower()
    5750      for a,b in [('&','and'),
    5751                  ('|','or'),
    5752                  ('~','not'),
    5753                  ('==','='),
    5754                  ('<','<'),
    5755                  ('>','>'),
    5756                  ('<=','<='),
    5757                  ('>=','>='),
    5758                  ('<>','!='),
    5759                  ('=<','<='),
    5760                  ('=>','>='),
    5761                  ('=','='),
    5762                  (' less or equal than ','<='),
    5763                  (' greater or equal than ','>='),
    5764                  (' equal or less than ','<='),
    5765                  (' equal or greater than ','>='),
    5766                  (' less or equal ','<='),
    5767                  (' greater or equal ','>='),
    5768                  (' equal or less ','<='),
    5769                  (' equal or greater ','>='),
    5770                  (' not equal to ','!='),
    5771                  (' not equal ','!='),
    5772                  (' equal to ','='),
    5773                  (' equal ','='),
    5774                  (' equals ','!='),
    5775                  (' less than ','<'),
    5776                  (' greater than ','>'),
    5777                  (' starts with ','startswith'),
    5778                  (' ends with ','endswith'),
    5779                  (' is ','=')]:
    5780          if a[0]==' ':
    5781              text = text.replace(' is'+a,' %s ' % b)
    5782          text = text.replace(a,' %s ' % b)
    5783      text = re.sub('\s+',' ',text).lower()
    5784      text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
    5785      query = field = neg = op = logic = None
    5786      for item in text.split():
    5787          if field is None:
    5788              if item == 'not':
    5789                  neg = True
    5790              elif not neg and not logic and item in ('and','or'):
    5791                  logic = item
    5792              elif item in field_map:
    5793                  field = field_map[item]
    5794              else:
    5795                  raise RuntimeError, "Invalid syntax"
    5796          elif not field is None and op is None:
    5797              op = item
    5798          elif not op is None:
    5799              if item.startswith('#'):
    5800                  if not item[1:] in constants:
    5801                      raise RuntimeError, "Invalid syntax"
    5802                  value = constants[item[1:]]
    5803              else:
    5804                  value = item
    5805                  if op == '=': op = 'like'
    5806              if op == '=': new_query = field==value
    5807              elif op == '<': new_query = field<value
    5808              elif op == '>': new_query = field>value
    5809              elif op == '<=': new_query = field<=value
    5810              elif op == '>=': new_query = field>=value
    5811              elif op == '!=': new_query = field!=value
    5812              elif field.type in ('text','string'):
    5813                  if op == 'contains': new_query = field.contains(value)
    5814                  elif op == 'like': new_query = field.like(value)
    5815                  elif op == 'startswith': new_query = field.startswith(value)
    5816                  elif op == 'endswith': new_query = field.endswith(value)
    5817                  else: raise RuntimeError, "Invalid operation"
    5818              else: raise RuntimeError, "Invalid operation"
    5819              if neg: new_query = ~new_query
    5820              if query is None:
    5821                  query = new_query
    5822              elif logic == 'and':
    5823                  query &= new_query
    5824              elif logic == 'or':
    5825                  query |= new_query
    5826              field = op = neg = logic = None
    5827      return query
    5828
    5829
    5830 -class DAL(dict):
    5831
    5832      """
    5833      an instance of this class represents a database connection
    5834
    5835      Example::
    5836
    5837         db = DAL('sqlite://test.db')
    5838         db.define_table('tablename', Field('fieldname1'),
    5839                                      Field('fieldname2'))
    5840      """
    5841
    5842      @staticmethod
    5843 -    def set_folder(folder):
    5844          """
    5845          # ## this allows gluon to set a folder for this thread
    5846          # ## <<<<<<<<< Should go away as new DAL replaces old sql.py
    5847          """
    5848          BaseAdapter.set_folder(folder)
    5849
    5850      @staticmethod
    5851 -    def distributed_transaction_begin(*instances):
    5852          if not instances:
    5853              return
    5854          thread_key = '%s.%s' % (socket.gethostname(), threading.currentThread())
    5855          keys = ['%s.%i' % (thread_key, i) for (i,db) in instances]
    5856          instances = enumerate(instances)
    5857          for (i, db) in instances:
    5858              if not db._adapter.support_distributed_transaction():
    5859                  raise SyntaxError, \
    5860                      'distributed transaction not suported by %s' % db._dbname
    5861          for (i, db) in instances:
    5862              db._adapter.distributed_transaction_begin(keys[i])
    5863
    5864      @staticmethod
    5865 -    def distributed_transaction_commit(*instances):
    5866          if not instances:
    5867              return
    5868          instances = enumerate(instances)
    5869          thread_key = '%s.%s' % (socket.gethostname(), threading.currentThread())
    5870          keys = ['%s.%i' % (thread_key, i) for (i,db) in instances]
    5871          for (i, db) in instances:
    5872              if not db._adapter.support_distributed_transaction():
    5873                  raise SyntaxError, \
    5874                      'distributed transaction not suported by %s' % db._dbanme
    5875          try:
    5876              for (i, db) in instances:
    5877                  db._adapter.prepare(keys[i])
    5878          except:
    5879              for (i, db) in instances:
    5880                  db._adapter.rollback_prepared(keys[i])
    5881              raise RuntimeError, 'failure to commit distributed transaction'
    5882          else:
    5883              for (i, db) in instances:
    5884                  db._adapter.commit_prepared(keys[i])
    5885          return
    5886
    5887
    5888 -    def __init__(self, uri='sqlite://dummy.db',
    5889                   pool_size=0, folder=None,
    5890                   db_codec='UTF-8', check_reserved=None,
    5891                   migrate=True, fake_migrate=False,
    5892                   migrate_enabled=True, fake_migrate_all=False,
    5893                   decode_credentials=False, driver_args=None,
    5894                   adapter_args=None, attempts=5, auto_import=False):
    5895          """
    5896          Creates a new Database Abstraction Layer instance.
    5897
    5898          Keyword arguments:
    5899
    5900          :uri: string that contains information for connecting to a database.
    5901                 (default: 'sqlite://dummy.db')
    5902          :pool_size: How many open connections to make to the database object.
    5903          :folder: <please update me>
    5904          :db_codec: string encoding of the database (default: 'UTF-8')
    5905          :check_reserved: list of adapters to check tablenames and column names
    5906                           against sql reserved keywords. (Default None)
    5907
    5908          * 'common' List of sql keywords that are common to all database types
    5909                  such as "SELECT, INSERT". (recommended)
    5910          * 'all' Checks against all known SQL keywords. (not recommended)
    5911                  <adaptername> Checks against the specific adapters list of keywords
    5912                  (recommended)
    5913          * '<adaptername>_nonreserved' Checks against the specific adapters
    5914                  list of nonreserved keywords. (if available)
    5915          :migrate (defaults to True) sets default migrate behavior for all tables
    5916          :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables
    5917          :migrate_enabled (defaults to True). If set to False disables ALL migrations
    5918          :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables
    5919          :attempts (defaults to 5). Number of times to attempt connecting
    5920          """
    5921          if not decode_credentials:
    5922              credential_decoder = lambda cred: cred
    5923          else:
    5924              credential_decoder = lambda cred: urllib.unquote(cred)
    5925          if folder:
    5926              self.set_folder(folder)
    5927          self._uri = uri
    5928          self._pool_size = pool_size
    5929          self._db_codec = db_codec
    5930          self._lastsql = ''
    5931          self._timings = []
    5932          self._pending_references = {}
    5933          self._request_tenant = 'request_tenant'
    5934          self._common_fields = []
    5935          self._referee_name = '%(table)s'
    5936          if not str(attempts).isdigit() or attempts < 0:
    5937              attempts = 5
    5938          if uri:
    5939              uris = isinstance(uri,(list,tuple)) and uri or [uri]
    5940              error = ''
    5941              connected = False
    5942              for k in range(attempts):
    5943                  for uri in uris:
    5944                      try:
    5945                          if is_jdbc and not uri.startswith('jdbc:'):
    5946                              uri = 'jdbc:'+uri
    5947                          self._dbname = regex_dbname.match(uri).group()
    5948                          if not self._dbname in ADAPTERS:
    5949                              raise SyntaxError, "Error in URI '%s' or database not supported" % self._dbname
    5950                          # notice that driver args or {} else driver_args
    5951                          # defaults to {} global, not correct
    5952                          args = (self,uri,pool_size,folder,
    5953                                  db_codec, credential_decoder,
    5954                                  driver_args or {}, adapter_args or {})
    5955                          self._adapter = ADAPTERS[self._dbname](*args)
    5956                          connected = True
    5957                          break
    5958                      except SyntaxError:
    5959                          raise
    5960                      except Exception, error:
    5961                          tb = traceback.format_exc()
    5962                          sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb))
    5963                  if connected:
    5964                      break
    5965                  else:
    5966                      time.sleep(1)
    5967              if not connected:
    5968                  raise RuntimeError, "Failure to connect, tried %d times:\n%s" % (attempts, tb)
    5969          else:
    5970              args = (self,'None',0,folder,db_codec)
    5971              self._adapter = BaseAdapter(*args)
    5972              migrate = fake_migrate = False
    5973          adapter = self._adapter
    5974          self._uri_hash = hashlib.md5(adapter.uri).hexdigest()
    5975          self._tables = SQLCallableList()
    5976          self.check_reserved = check_reserved
    5977          if self.check_reserved:
    5978              from reserved_sql_keywords import ADAPTERS as RSK
    5979              self.RSK = RSK
    5980          self._migrate = migrate
    5981          self._fake_migrate = fake_migrate
    5982          self._migrate_enabled = migrate_enabled
    5983          self._fake_migrate_all = fake_migrate_all
    5984          if auto_import:
    5985              self.import_table_definitions(adapter.folder)
    5986
    5987      @property
    5988 -    def tables(self):
    5989          return self._tables
    5990
    5991 -    def import_table_definitions(self,path,migrate=False,fake_migrate=False):
    5992          pattern = os.path.join(path,self._uri_hash+'_*.table')
    5993          for filename in glob.glob(pattern):
    5994              tfile = self._adapter.file_open(filename, 'r')
    5995              try:
    5996                  sql_fields = cPickle.load(tfile)
    5997                  name = filename[len(pattern)-7:-6]
    5998                  mf = [(value['sortable'],Field(key,type=value['type'])) \
    5999                            for key, value in sql_fields.items()]
    6000                  mf.sort(lambda a,b: cmp(a[0],b[0]))
    6001                  self.define_table(name,*[item[1] for item in mf],
    6002                                    **dict(migrate=migrate,fake_migrate=fake_migrate))
    6003              finally:
    6004                  self._adapter.file_close(tfile)
    6005
    6006 -    def check_reserved_keyword(self, name):
    6007          """
    6008          Validates ``name`` against SQL keywords
    6009          Uses self.check_reserve which is a list of
    6010          operators to use.
    6011          self.check_reserved
    6012          ['common', 'postgres', 'mysql']
    6013          self.check_reserved
    6014          ['all']
    6015          """
    6016          for backend in self.check_reserved:
    6017              if name.upper() in self.RSK[backend]:
    6018                  raise SyntaxError, 'invalid table/column name "%s" is a "%s" reserved SQL keyword' % (name, backend.upper())
    6019
    6020 -    def __contains__(self, tablename):
    6021          if self.has_key(tablename):
    6022              return True
    6023          else:
    6024              return False
    6025
    6026 -    def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
    6027          """
    6028          EXAMPLE:
    6029
    6030  db.define_table('person',Field('name'),Field('info'))
    6031  db.define_table('pet',Field('owner',db.person),Field('name'),Field('info'))
    6032
    6033  @request.restful()
    6034  def index():
    6035      def GET(*args,**vars):
    6036          patterns = [
    6037              "/friends[person]",
    6038              "/{friend.name.startswith}",
    6039              "/{friend.name}/:field",
    6040              "/{friend.name}/pets[pet.owner]",
    6041              "/{friend.name}/pet[pet.owner]/{pet.name}",
    6042              "/{friend.name}/pet[pet.owner]/{pet.name}/:field"
    6043              ]
    6044          parser = db.parse_as_rest(patterns,args,vars)
    6045          if parser.status == 200:
    6046              return dict(content=parser.response)
    6047          else:
    6048              raise HTTP(parser.status,parser.error)
    6049      def POST(table_name,**vars):
    6050          if table_name == 'person':
    6051              return db.person.validate_and_insert(**vars)
    6052          elif table_name == 'pet':
    6053              return db.pet.validate_and_insert(**vars)
    6054          else:
    6055              raise HTTP(400)
    6056      return locals()
    6057          """
    6058
    6059          db = self
    6060          re1 = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
    6061          re2 = re.compile('^.+\[.+\]$')
    6062
    6063          def auto_table(table,base='',depth=0):
    6064              patterns = []
    6065              for field in db[table].fields:
    6066                  if base:
    6067                      tag = '%s/%s' % (base,field.replace('_','-'))
    6068                  else:
    6069                      tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-'))
    6070                  f = db[table][field]
    6071                  if not f.readable: continue
    6072                  if f.type=='id' or 'slug' in field or f.type.startswith('reference'):
    6073                      tag += '/{%s.%s}' % (table,field)
    6074                      patterns.append(tag)
    6075                      patterns.append(tag+'/:field')
    6076                  elif f.type.startswith('boolean'):
    6077                      tag += '/{%s.%s}' % (table,field)
    6078                      patterns.append(tag)
    6079                      patterns.append(tag+'/:field')
    6080                  elif f.type.startswith('double') or f.type.startswith('integer'):
    6081                      tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field)
    6082                      patterns.append(tag)
    6083                      patterns.append(tag+'/:field')
    6084                  elif f.type.startswith('list:'):
    6085                      tag += '/{%s.%s.contains}' % (table,field)
    6086                      patterns.append(tag)
    6087                      patterns.append(tag+'/:field')
    6088                  elif f.type in ('date','datetime'):
    6089                      tag+= '/{%s.%s.year}' % (table,field)
    6090                      patterns.append(tag)
    6091                      patterns.append(tag+'/:field')
    6092                      tag+='/{%s.%s.month}' % (table,field)
    6093                      patterns.append(tag)
    6094                      patterns.append(tag+'/:field')
    6095                      tag+='/{%s.%s.day}' % (table,field)
    6096                      patterns.append(tag)
    6097                      patterns.append(tag+'/:field')
    6098                  if f.type in ('datetime','time'):
    6099                      tag+= '/{%s.%s.hour}' % (table,field)
    6100                      patterns.append(tag)
    6101                      patterns.append(tag+'/:field')
    6102                      tag+='/{%s.%s.minute}' % (table,field)
    6103                      patterns.append(tag)
    6104                      patterns.append(tag+'/:field')
    6105                      tag+='/{%s.%s.second}' % (table,field)
    6106                      patterns.append(tag)
    6107                      patterns.append(tag+'/:field')
    6108                  if depth>0:
    6109                      for rtable,rfield in db[table]._referenced_by:
    6110                          tag+='/%s[%s.%s]' % (rtable,rtable,rfield)
    6111                          patterns.append(tag)
    6112                          patterns += auto_table(rtable,base=tag,depth=depth-1)
    6113              return patterns
    6114
    6115          if patterns=='auto':
    6116              patterns=[]
    6117              for table in db.tables:
    6118                  if not table.startswith('auth_'):
    6119                      patterns.append('/%s[%s]' % (table,table))
    6120                      patterns += auto_table(table,base='',depth=1)
    6121          else:
    6122              i = 0
    6123              while i<len(patterns):
    6124                  pattern = patterns[i]
    6125                  tokens = pattern.split('/')
    6126                  if tokens[-1].startswith(':auto') and re2.match(tokens[-1]):
    6127                      new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1],
    6128                                                '/'.join(tokens[:-1]))
    6129                      patterns = patterns[:i]+new_patterns+patterns[i+1:]
    6130                      i += len(new_patterns)
    6131                  else:
    6132                      i += 1
    6133          if '/'.join(args) == 'patterns':
    6134              return Row({'status':200,'pattern':'list',
    6135                          'error':None,'response':patterns})
    6136          for pattern in patterns:
    6137              otable=table=None
    6138              if not isinstance(queries,dict):
    6139                  dbset=db(queries)
    6140              i=0
    6141              tags = pattern[1:].split('/')
    6142              if len(tags)!=len(args):
    6143                  continue
    6144              for tag in tags:
    6145                  if re1.match(tag):
    6146                      # print 're1:'+tag
    6147                      tokens = tag[1:-1].split('.')
    6148                      table, field = tokens[0], tokens[1]
    6149                      if not otable or table == otable:
    6150                          if len(tokens)==2 or tokens[2]=='eq':
    6151                              query = db[table][field]==args[i]
    6152                          elif tokens[2]=='ne':
    6153                              query = db[table][field]!=args[i]
    6154                          elif tokens[2]=='lt':
    6155                              query = db[table][field]<args[i]
    6156                          elif tokens[2]=='gt':
    6157                              query = db[table][field]>args[i]
    6158                          elif tokens[2]=='ge':
    6159                              query = db[table][field]>=args[i]
    6160                          elif tokens[2]=='le':
    6161                              query = db[table][field]<=args[i]
    6162                          elif tokens[2]=='year':
    6163                              query = db[table][field].year()==args[i]
    6164                          elif tokens[2]=='month':
    6165                              query = db[table][field].month()==args[i]
    6166                          elif tokens[2]=='day':
    6167                              query = db[table][field].day()==args[i]
    6168                          elif tokens[2]=='hour':
    6169                              query = db[table][field].hour()==args[i]
    6170                          elif tokens[2]=='minute':
    6171                              query = db[table][field].minutes()==args[i]
    6172                          elif tokens[2]=='second':
    6173                              query = db[table][field].seconds()==args[i]
    6174                          elif tokens[2]=='startswith':
    6175                              query = db[table][field].startswith(args[i])
    6176                          elif tokens[2]=='contains':
    6177                              query = db[table][field].contains(args[i])
    6178                          else:
    6179                              raise RuntimeError, "invalid pattern: %s" % pattern
    6180                          if len(tokens)==4 and tokens[3]=='not':
    6181                              query = ~query
    6182                          elif len(tokens)>=4:
    6183                              raise RuntimeError, "invalid pattern: %s" % pattern
    6184                          if not otable and isinstance(queries,dict):
    6185                              dbset = db(queries[table])
    6186                          dbset=dbset(query)
    6187                      else:
    6188                          raise RuntimeError, "missing relation in pattern: %s" % pattern
    6189                  elif re2.match(tag) and args[i]==tag[:tag.find('[')]:
    6190                      ref = tag[tag.find('[')+1:-1]
    6191                      if '.' in ref and otable:
    6192                          table,field = ref.split('.')
    6193                          # print table,field
    6194                          if nested_select:
    6195                              try:
    6196                                  dbset=db(db[table][field].belongs(dbset._select(db[otable]._id)))
    6197                              except ValueError:
    6198                                  return Row({'status':400,'pattern':pattern,
    6199                                              'error':'invalid path','response':None})
    6200                          else:
    6201                              items = [item.id for item in dbset.select(db[otable]._id)]
    6202                              dbset=db(db[table][field].belongs(items))
    6203                      else:
    6204                          table = ref
    6205                          if not otable and isinstance(queries,dict):
    6206                              dbset = db(queries[table])
    6207                          dbset=dbset(db[table])
    6208                  elif tag==':field' and table:
    6209                      # # print 're3:'+tag
    6210                      field = args[i]
    6211                      if not field in db[table]: break
    6212                      try:
    6213                          item =  dbset.select(db[table][field],limitby=(0,1)).first()
    6214                      except ValueError:
    6215                          return Row({'status':400,'pattern':pattern,
    6216                                      'error':'invalid path','response':None})
    6217                      if not item:
    6218                          return Row({'status':404,'pattern':pattern,
    6219                                      'error':'record not found','response':None})
    6220                      else:
    6221                          return Row({'status':200,'response':item[field],
    6222                                      'pattern':pattern})
    6223                  elif tag != args[i]:
    6224                      break
    6225                  otable = table
    6226                  i += 1
    6227                  if i==len(tags) and table:
    6228                      ofields = vars.get('order',db[table]._id.name).split('|')
    6229                      try:
    6230                          orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields]
    6231                      except KeyError:
    6232                          return Row({'status':400,'error':'invalid orderby','response':None})
    6233                      fields = [field for field in db[table] if field.readable]
    6234                      count = dbset.count()
    6235                      try:
    6236                          offset = int(vars.get('offset',None) or 0)
    6237                          limits = (offset,int(vars.get('limit',None) or 1000)+offset)
    6238                      except ValueError:
    6239                          Row({'status':400,'error':'invalid limits','response':None})
    6240                      if count > limits[1]-limits[0]:
    6241                          Row({'status':400,'error':'too many records','response':None})
    6242                      try:
    6243                          response = dbset.select(limitby=limits,orderby=orderby,*fields)
    6244                      except ValueError:
    6245                          return Row({'status':400,'pattern':pattern,
    6246                                      'error':'invalid path','response':None})
    6247                      return Row({'status':200,'response':response,'pattern':pattern})
    6248          return Row({'status':400,'error':'no matching pattern','response':None})
    6249
    6250
    6251 -    def define_table(
    6252          self,
    6253          tablename,
    6254          *fields,
    6255          **args
    6256          ):
    6257
    6258          for key in args:
    6259              if key not in [
    6260                      'migrate',
    6261                      'primarykey',
    6262                      'fake_migrate',
    6263                      'format',
    6264                      'singular',
    6265                      'plural',
    6266                      'trigger_name',
    6267                      'sequence_name',
    6268                      'common_filter',
    6269                      'polymodel',
    6270                      'table_class']:
    6271                  raise SyntaxError, 'invalid table "%s" attribute: %s' \
    6272                      % (tablename, key)
    6273          if not isinstance(tablename,str):
    6274              raise SyntaxError, "missing table name"
    6275          tablename = cleanup(tablename)
    6276          migrate = self._migrate_enabled and args.get('migrate',
    6277                                                       self._migrate)
    6278          fake_migrate = self._fake_migrate_all or args.get('fake_migrate',
    6279                                                            self._fake_migrate)
    6280          table_class = args.get('table_class',Table)
    6281          format = args.get('format',None)
    6282          trigger_name = args.get('trigger_name', None)
    6283          sequence_name = args.get('sequence_name', None)
    6284          primarykey =args.get('primarykey',None)
    6285          polymodel = args.get('polymodel',None)
    6286          singular = args.get('singular',tablename.replace('_',' ').capitalize())
    6287          plural = args.get('plural',pluralize(singular.lower()).capitalize())
    6288          lowertablename = tablename.lower()
    6289
    6290          if tablename.startswith('_') or hasattr(self,lowertablename) or \
    6291                  regex_python_keywords.match(tablename):
    6292              raise SyntaxError, 'invalid table name: %s' % tablename
    6293          elif lowertablename in self.tables:
    6294              raise SyntaxError, 'table already defined: %s' % tablename
    6295          elif self.check_reserved:
    6296              self.check_reserved_keyword(tablename)
    6297
    6298          if self._common_fields:
    6299              fields = [f for f in fields] + [f for f in self._common_fields]
    6300
    6301          common_filter = args.get('common_filter', None)
    6302
    6303          t = self[tablename] = table_class(self, tablename, *fields,
    6304                                            **dict(primarykey=primarykey,
    6305                                                   trigger_name=trigger_name,
    6306                                                   sequence_name=sequence_name,
    6307                                                   common_filter=common_filter))
    6308
    6309          # db magic
    6310          if self._uri in (None,'None'):
    6311              return t
    6312
    6313          t._create_references()
    6314
    6315          if migrate or self._adapter.dbengine=='google:datastore':
    6316              try:
    6317                  sql_locker.acquire()
    6318                  self._adapter.create_table(t,migrate=migrate,
    6319                                             fake_migrate=fake_migrate,
    6320                                             polymodel=polymodel)
    6321              finally:
    6322                  sql_locker.release()
    6323          else:
    6324              t._dbt = None
    6325          self.tables.append(tablename)
    6326          t._format = format
    6327          t._singular = singular
    6328          t._plural = plural
    6329          t._actual = True
    6330          return t
    6331
    6332 -    def __iter__(self):
    6333          for tablename in self.tables:
    6334              yield self[tablename]
    6335
    6336 -    def __getitem__(self, key):
    6337          return dict.__getitem__(self, str(key))
    6338
    6339 -    def __setitem__(self, key, value):
    6340          dict.__setitem__(self, str(key), value)
    6341
    6342 -    def __getattr__(self, key):
    6343          return self[key]
    6344
    6345 -    def __setattr__(self, key, value):
    6346          if key[:1]!='_' and key in self:
    6347              raise SyntaxError, \
    6348                  'Object %s exists and cannot be redefined' % key
    6349          self[key] = value
    6350
    6351 -    def __repr__(self):
    6352          return '<DAL ' + dict.__repr__(self) + '>'
    6353
    6354 -    def smart_query(self,fields,text):
    6355          return Set(self, smart_query(fields,text))
    6356
    6357 -    def __call__(self, query=None, ignore_common_filters=None):
    6358          if isinstance(query,Table):
    6359              query = query._id>0
    6360          elif isinstance(query,Field):
    6361              query = query!=None
    6362          return Set(self, query, ignore_common_filters=ignore_common_filters)
    6363
    6364 -    def commit(self):
    6365          self._adapter.commit()
    6366
    6367 -    def rollback(self):
    6368          self._adapter.rollback()
    6369
    6370 -    def executesql(self, query, placeholders=None, as_dict=False):
    6371          """
    6372          placeholders is optional and will always be None when using DAL.
    6373          If using raw SQL with placeholders, placeholders may be
    6374          a sequence of values to be substituted in
    6375          or, (if supported by the DB driver), a dictionary with keys
    6376          matching named placeholders in your SQL.
    6377
    6378          Added 2009-12-05 "as_dict" optional argument. Will always be
    6379          None when using DAL. If using raw SQL can be set to True
    6380          and the results cursor returned by the DB driver will be
    6381          converted to a sequence of dictionaries keyed with the db
    6382          field names. Tested with SQLite but should work with any database
    6383          since the cursor.description used to get field names is part of the
    6384          Python dbi 2.0 specs. Results returned with as_dict=True are
    6385          the same as those returned when applying .to_list() to a DAL query.
    6386
    6387          [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}]
    6388
    6389          --bmeredyk
    6390          """
    6391          if placeholders:
    6392              self._adapter.execute(query, placeholders)
    6393          else:
    6394              self._adapter.execute(query)
    6395          if as_dict:
    6396              if not hasattr(self._adapter.cursor,'description'):
    6397                  raise RuntimeError, "database does not support executesql(...,as_dict=True)"
    6398              # Non-DAL legacy db query, converts cursor results to dict.
    6399              # sequence of 7-item sequences. each sequence tells about a column.
    6400              # first item is always the field name according to Python Database API specs
    6401              columns = self._adapter.cursor.description
    6402              # reduce the column info down to just the field names
    6403              fields = [f[0] for f in columns]
    6404              # will hold our finished resultset in a list
    6405              data = self._adapter.cursor.fetchall()
    6406              # convert the list for each row into a dictionary so it's
    6407              # easier to work with. row['field_name'] rather than row[0]
    6408              return [dict(zip(fields,row)) for row in data]
    6409          # see if any results returned from database
    6410          try:
    6411              return self._adapter.cursor.fetchall()
    6412          except:
    6413              return None
    6414
    6415 -    def _update_referenced_by(self, other):
    6416          for tablename in self.tables:
    6417              by = self[tablename]._referenced_by
    6418              by[:] = [item for item in by if not item[0] == other]
    6419
    6420 -    def export_to_csv_file(self, ofile, *args, **kwargs):
    6421          step = int(kwargs.get('max_fetch_rows,',500))
    6422          write_colnames = kwargs['write_colnames'] = \
    6423              kwargs.get("write_colnames", True)
    6424          for table in self.tables:
    6425              ofile.write('TABLE %s\r\n' % table)
    6426              query = self[table]._id > 0
    6427              nrows = self(query).count()
    6428              kwargs['write_colnames'] = write_colnames
    6429              for k in range(0,nrows,step):
    6430                  self(query).select(limitby=(k,k+step)).export_to_csv_file(
    6431                      ofile, *args, **kwargs)
    6432                  kwargs['write_colnames'] = False
    6433              ofile.write('\r\n\r\n')
    6434          ofile.write('END')
    6435
    6436 -    def import_from_csv_file(self, ifile, id_map=None, null='<NULL>',
    6437                               unique='uuid', *args, **kwargs):
    6438          if id_map is None: id_map={}
    6439          for line in ifile:
    6440              line = line.strip()
    6441              if not line:
    6442                  continue
    6443              elif line == 'END':
    6444                  return
    6445              elif not line.startswith('TABLE ') or not line[6:] in self.tables:
    6446                  raise SyntaxError, 'invalid file format'
    6447              else:
    6448                  tablename = line[6:]
    6449                  self[tablename].import_from_csv_file(ifile, id_map, null,
    6450                                                       unique, *args, **kwargs)
    6451
    6452 -class SQLALL(object):
    6453      """
    6454      Helper class providing a comma-separated string having all the field names
    6455      (prefixed by table name and '.')
    6456
    6457      normally only called from within gluon.sql
    6458      """
    6459
    6460 -    def __init__(self, table):
    6461          self.table = table
    6462
    6463 -    def __str__(self):
    6464          return ', '.join([str(field) for field in self.table])
    6465
    6466
    6467 -class Reference(int):
    6468
    6469 -    def __allocate(self):
    6470          if not self._record:
    6471              self._record = self._table[int(self)]
    6472          if not self._record:
    6473              raise RuntimeError, "Using a recursive select but encountered a broken reference: %s %d"%(self._table, int(self))
    6474
    6475 -    def __getattr__(self, key):
    6476          if key == 'id':
    6477              return int(self)
    6478          self.__allocate()
    6479          return self._record.get(key, None)
    6480
    6481 -    def get(self, key):
    6482          return self.__getattr__(key)
    6483
    6484 -    def __setattr__(self, key, value):
    6485          if key.startswith('_'):
    6486              int.__setattr__(self, key, value)
    6487              return
    6488          self.__allocate()
    6489          self._record[key] =  value
    6490
    6491 -    def __getitem__(self, key):
    6492          if key == 'id':
    6493              return int(self)
    6494          self.__allocate()
    6495          return self._record.get(key, None)
    6496
    6497 -    def __setitem__(self,key,value):
    6498          self.__allocate()
    6499          self._record[key] = value
    6500
    6501
    6502 -def Reference_unpickler(data):
    6503      return marshal.loads(data)
    6504
    6505 -def Reference_pickler(data):
    6506      try:
    6507          marshal_dump = marshal.dumps(int(data))
    6508      except AttributeError:
    6509          marshal_dump = 'i%s' % struct.pack('<i', int(data))
    6510      return (Reference_unpickler, (marshal_dump,))
    6511
    6512  copy_reg.pickle(Reference, Reference_pickler, Reference_unpickler)
    6513
    6514
    6515 -class Table(dict):
    6516
    6517      """
    6518      an instance of this class represents a database table
    6519
    6520      Example::
    6521
    6522          db = DAL(...)
    6523          db.define_table('users', Field('name'))
    6524          db.users.insert(name='me') # print db.users._insert(...) to see SQL
    6525          db.users.drop()
    6526      """
    6527
    6528 -    def __init__(
    6529          self,
    6530          db,
    6531          tablename,
    6532          *fields,
    6533          **args
    6534          ):
    6535          """
    6536          Initializes the table and performs checking on the provided fields.
    6537
    6538          Each table will have automatically an 'id'.
    6539
    6540          If a field is of type Table, the fields (excluding 'id') from that table
    6541          will be used instead.
    6542
    6543          :raises SyntaxError: when a supplied field is of incorrect type.
    6544          """
    6545          self._actual = False # set to True by define_table()
    6546          self._tablename = tablename
    6547          self._sequence_name = args.get('sequence_name',None) or \
    6548              db and db._adapter.sequence_name(tablename)
    6549          self._trigger_name = args.get('trigger_name',None) or \
    6550              db and db._adapter.trigger_name(tablename)
    6551          self._common_filter = args.get('common_filter', None)
    6552          primarykey = args.get('primarykey', None)
    6553          fieldnames,newfields=set(),[]
    6554          if primarykey:
    6555              if not isinstance(primarykey,list):
    6556                  raise SyntaxError, \
    6557                      "primarykey must be a list of fields from table '%s'" \
    6558                      % tablename
    6559              self._primarykey = primarykey
    6560          elif not [f for f in fields if isinstance(f,Field) and f.type=='id']:
    6561              field = Field('id', 'id')
    6562              newfields.append(field)
    6563              fieldnames.add('id')
    6564              self._id = field
    6565          for field in fields:
    6566              if not isinstance(field, (Field, Table)):
    6567                  raise SyntaxError, \
    6568                      'define_table argument is not a Field or Table: %s' % field
    6569              elif isinstance(field, Field) and not field.name in fieldnames:
    6570                  if hasattr(field, '_db'):
    6571                      field = copy.copy(field)
    6572                  newfields.append(field)
    6573                  fieldnames.add(field.name)
    6574                  if field.type=='id':
    6575                      self._id = field
    6576              elif isinstance(field, Table):
    6577                  table = field
    6578                  for field in table:
    6579                      if not field.name in fieldnames and not field.type=='id':
    6580                          field = copy.copy(field)
    6581                          # correct self references
    6582                          if not table._actual and field.type == 'reference '+table._tablename:
    6583                              field.type = 'reference '+self._tablename
    6584                          newfields.append(field)
    6585                          fieldnames.add(field.name)
    6586              else:
    6587                  # let's ignore new fields with duplicated names!!!
    6588                  pass
    6589          fields = newfields
    6590          self._db = db
    6591          tablename = tablename
    6592          self._fields = SQLCallableList()
    6593          self.virtualfields = []
    6594          fields = list(fields)
    6595
    6596          if db and self._db._adapter.uploads_in_blob==True:
    6597              for field in fields:
    6598                  if isinstance(field, Field) and field.type == 'upload'\
    6599                          and field.uploadfield is True:
    6600                      tmp = field.uploadfield = '%s_blob' % field.name
    6601                      fields.append(self._db.Field(tmp, 'blob', default=''))
    6602
    6603          lower_fieldnames = set()
    6604          reserved = dir(Table) + ['fields']
    6605          for field in fields:
    6606              if db and db.check_reserved:
    6607                  db.check_reserved_keyword(field.name)
    6608              elif field.name in reserved:
    6609                  raise SyntaxError, "field name %s not allowed" % field.name
    6610
    6611              if field.name.lower() in lower_fieldnames:
    6612                  raise SyntaxError, "duplicate field %s in table %s" \
    6613                      % (field.name, tablename)
    6614              else:
    6615                  lower_fieldnames.add(field.name.lower())
    6616
    6617              self.fields.append(field.name)
    6618              self[field.name] = field
    6619              if field.type == 'id':
    6620                  self['id'] = field
    6621              field.tablename = field._tablename = tablename
    6622              field.table = field._table = self
    6623              field.db = field._db = self._db
    6624              if self._db and not field.type in ('text','blob') and \
    6625                      self._db._adapter.maxcharlength < field.length:
    6626                  field.length = self._db._adapter.maxcharlength
    6627              if field.requires is DEFAULT:
    6628                  field.requires = sqlhtml_validators(field)
    6629          self.ALL = SQLALL(self)
    6630
    6631          if hasattr(self,'_primarykey'):
    6632              for k in self._primarykey:
    6633                  if k not in self.fields:
    6634                      raise SyntaxError, \
    6635                          "primarykey must be a list of fields from table '%s " % tablename
    6636                  else:
    6637                      self[k].notnull = True
    6638
    6639      @property
    6640 -    def fields(self):
    6641          return self._fields
    6642
    6643 -    def update(self,*args,**kwargs):
    6644          raise RuntimeError, "Syntax Not Supported"
    6645
    6646 -    def _validate(self,**vars):
    6647          errors = Row()
    6648          for key,value in vars.items():
    6649              value,error = self[key].validate(value)
    6650              if error:
    6651                  errors[key] = error
    6652          return errors
    6653
    6654 -    def _create_references(self):
    6655          pr = self._db._pending_references
    6656          self._referenced_by = []
    6657          for fieldname in self.fields:
    6658              field=self[fieldname]
    6659              if isinstance(field.type,str) and field.type[:10] == 'reference ':
    6660                  ref = field.type[10:].strip()
    6661                  if not ref.split():
    6662                      raise SyntaxError, 'Table: reference to nothing: %s' %ref
    6663                  refs = ref.split('.')
    6664                  rtablename = refs[0]
    6665                  if not rtablename in self._db:
    6666                      pr[rtablename] = pr.get(rtablename,[]) + [field]
    6667                      continue
    6668                  rtable = self._db[rtablename]
    6669                  if len(refs)==2:
    6670                      rfieldname = refs[1]
    6671                      if not hasattr(rtable,'_primarykey'):
    6672                          raise SyntaxError,\
    6673                              'keyed tables can only reference other keyed tables (for now)'
    6674                      if rfieldname not in rtable.fields:
    6675                          raise SyntaxError,\
    6676                              "invalid field '%s' for referenced table '%s' in table '%s'" \
    6677                              % (rfieldname, rtablename, self._tablename)
    6678                  rtable._referenced_by.append((self._tablename, field.name))
    6679          for referee in pr.get(self._tablename,[]):
    6680              self._referenced_by.append((referee._tablename,referee.name))
    6681
    6682 -    def _filter_fields(self, record, id=False):
    6683          return dict([(k, v) for (k, v) in record.items() if k
    6684                       in self.fields and (self[k].type!='id' or id)])
    6685
    6686 -    def _build_query(self,key):
    6687          """ for keyed table only """
    6688          query = None
    6689          for k,v in key.iteritems():
    6690              if k in self._primarykey:
    6691                  if query:
    6692                      query = query & (self[k] == v)
    6693                  else:
    6694                      query = (self[k] == v)
    6695              else:
    6696                  raise SyntaxError, \
    6697                  'Field %s is not part of the primary key of %s' % \
    6698                  (k,self._tablename)
    6699          return query
    6700
    6701 -    def __getitem__(self, key):
    6702          if not key:
    6703              return None
    6704          elif isinstance(key, dict):
    6705              """ for keyed table """
    6706              query = self._build_query(key)
    6707              rows = self._db(query).select()
    6708              if rows:
    6709                  return rows[0]
    6710              return None
    6711          elif str(key).isdigit():
    6712              return self._db(self._id == key).select(limitby=(0,1)).first()
    6713          elif key:
    6714              return dict.__getitem__(self, str(key))
    6715
    6716 -    def __call__(self, key=DEFAULT, **kwargs):
    6717          for_update = kwargs.get('_for_update',False)
    6718          if '_for_update' in kwargs: del kwargs['_for_update']
    6719          if not key is DEFAULT:
    6720              if isinstance(key, Query):
    6721                  record = self._db(key).select(
    6722                      limitby=(0,1),for_update=for_update).first()
    6723              elif not str(key).isdigit():
    6724                  record = None
    6725              else:
    6726                  record = self._db(self._id == key).select(
    6727                      limitby=(0,1),for_update=for_update).first()
    6728              if record:
    6729                  for k,v in kwargs.items():
    6730                      if record[k]!=v: return None
    6731              return record
    6732          elif kwargs:
    6733              query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.items()])
    6734              return self._db(query).select(limitby=(0,1),for_update=for_update).first()
    6735          else:
    6736              return None
    6737
    6738 -    def __setitem__(self, key, value):
    6739          if isinstance(key, dict) and isinstance(value, dict):
    6740              """ option for keyed table """
    6741              if set(key.keys()) == set(self._primarykey):
    6742                  value = self._filter_fields(value)
    6743                  kv = {}
    6744                  kv.update(value)
    6745                  kv.update(key)
    6746                  if not self.insert(**kv):
    6747                      query = self._build_query(key)
    6748                      self._db(query).update(**self._filter_fields(value))
    6749              else:
    6750                  raise SyntaxError,\
    6751                      'key must have all fields from primary key: %s'%\
    6752                      (self._primarykey)
    6753          elif str(key).isdigit():
    6754              if key == 0:
    6755                  self.insert(**self._filter_fields(value))
    6756              elif self._db(self._id == key)\
    6757                      .update(**self._filter_fields(value)) is None:
    6758                  raise SyntaxError, 'No such record: %s' % key
    6759          else:
    6760              if isinstance(key, dict):
    6761                  raise SyntaxError,\
    6762                      'value must be a dictionary: %s' % value
    6763              dict.__setitem__(self, str(key), value)
    6764
    6765 -    def __delitem__(self, key):
    6766          if isinstance(key, dict):
    6767              query = self._build_query(key)
    6768              if not self._db(query).delete():
    6769                  raise SyntaxError, 'No such record: %s' % key
    6770          elif not str(key).isdigit() or not self._db(self._id == key).delete():
    6771              raise SyntaxError, 'No such record: %s' % key
    6772
    6773 -    def __getattr__(self, key):
    6774          return self[key]
    6775
    6776 -    def __setattr__(self, key, value):
    6777          if key[:1]!='_' and key in self:
    6778              raise SyntaxError, 'Object exists and cannot be redefined: %s' % key
    6779          self[key] = value
    6780
    6781 -    def __iter__(self):
    6782          for fieldname in self.fields:
    6783              yield self[fieldname]
    6784
    6785 -    def __repr__(self):
    6786          return '<Table ' + dict.__repr__(self) + '>'
    6787
    6788 -    def __str__(self):
    6789          if self.get('_ot', None):
    6790              return '%s AS %s' % (self._ot, self._tablename)
    6791          return self._tablename
    6792
    6793 -    def _drop(self, mode = ''):
    6794          return self._db._adapter._drop(self, mode)
    6795
    6796 -    def drop(self, mode = ''):
    6797          return self._db._adapter.drop(self,mode)
    6798
    6799 -    def _listify(self,fields,update=False):
    6800          new_fields = []
    6801          new_fields_names = []
    6802          for name in fields:
    6803              if not name in self.fields:
    6804                  if name != 'id':
    6805                      raise SyntaxError, 'Field %s does not belong to the table' % name
    6806              else:
    6807                  new_fields.append((self[name],fields[name]))
    6808                  new_fields_names.append(name)
    6809          for ofield in self:
    6810              if not ofield.name in new_fields_names:
    6811                  if not update and not ofield.default is None:
    6812                      new_fields.append((ofield,ofield.default))
    6813                  elif update and not ofield.update is None:
    6814                      new_fields.append((ofield,ofield.update))
    6815          for ofield in self:
    6816              if not ofield.name in new_fields_names and ofield.compute:
    6817                  try:
    6818                      new_fields.append((ofield,ofield.compute(Row(fields))))
    6819                  except KeyError:
    6820                      pass
    6821              if not update and ofield.required and not ofield.name in new_fields_names:
    6822                  raise SyntaxError,'Table: missing required field: %s' % ofield.name
    6823          return new_fields
    6824
    6825 -    def _insert(self, **fields):
    6826          return self._db._adapter._insert(self,self._listify(fields))
    6827
    6828 -    def insert(self, **fields):
    6829          return self._db._adapter.insert(self,self._listify(fields))
    6830
    6831 -    def validate_and_insert(self,**fields):
    6832          response = Row()
    6833          response.errors = Row()
    6834          new_fields = copy.copy(fields)
    6835          for key,value in fields.items():
    6836              value,error = self[key].validate(value)
    6837              if error:
    6838                  response.errors[key] = error
    6839              else:
    6840                  new_fields[key] = value
    6841          if not response.errors:
    6842              response.id = self.insert(**new_fields)
    6843          else:
    6844              response.id = None
    6845          return response
    6846
    6847 -    def update_or_insert(self, _key=DEFAULT, **values):
    6848          if _key is DEFAULT:
    6849              record = self(**values)
    6850          else:
    6851              record = self(_key)
    6852          if record:
    6853              record.update_record(**values)
    6854              newid = None
    6855          else:
    6856              newid = self.insert(**values)
    6857          return newid
    6858
    6859 -    def bulk_insert(self, items):
    6860          """
    6861          here items is a list of dictionaries
    6862          """
    6863          items = [self._listify(item) for item in items]
    6864          return self._db._adapter.bulk_insert(self,items)
    6865
    6866 -    def _truncate(self, mode = None):
    6867          return self._db._adapter._truncate(self, mode)
    6868
    6869 -    def truncate(self, mode = None):
    6870          return self._db._adapter.truncate(self, mode)
    6871
    6872 -    def import_from_csv_file(
    6873          self,
    6874          csvfile,
    6875          id_map=None,
    6876          null='<NULL>',
    6877          unique='uuid',
    6878          *args, **kwargs
    6879          ):
    6880          """
    6881          import records from csv file. Column headers must have same names as
    6882          table fields. field 'id' is ignored. If column names read 'table.file'
    6883          the 'table.' prefix is ignored.
    6884          'unique' argument is a field which must be unique
    6885          (typically a uuid field)
    6886          """
    6887
    6888          delimiter = kwargs.get('delimiter', ',')
    6889          quotechar = kwargs.get('quotechar', '"')
    6890          quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
    6891
    6892          reader = csv.reader(csvfile, delimiter=delimiter, quotechar=quotechar, quoting=quoting)
    6893          colnames = None
    6894          if isinstance(id_map, dict):
    6895              if not self._tablename in id_map:
    6896                  id_map[self._tablename] = {}
    6897              id_map_self = id_map[self._tablename]
    6898
    6899          def fix(field, value, id_map):
    6900              list_reference_s='list:reference'
    6901              if value == null:
    6902                  value = None
    6903              elif field.type=='blob':
    6904                  value = base64.b64decode(value)
    6905              elif field.type=='double':
    6906                  if not value.strip():
    6907                      value = None
    6908                  else:
    6909                      value = float(value)
    6910              elif field.type=='integer':
    6911                  if not value.strip():
    6912                      value = None
    6913                  else:
    6914                      value = int(value)
    6915              elif field.type.startswith('list:string'):
    6916                  value = bar_decode_string(value)
    6917              elif field.type.startswith(list_reference_s):
    6918                  ref_table = field.type[len(list_reference_s):].strip()
    6919                  value = [id_map[ref_table][int(v)] \
    6920                               for v in bar_decode_string(value)]
    6921              elif field.type.startswith('list:'):
    6922                  value = bar_decode_integer(value)
    6923              elif id_map and field.type.startswith('reference'):
    6924                  try:
    6925                      value = id_map[field.type[9:].strip()][int(value)]
    6926                  except KeyError:
    6927                      pass
    6928              return (field.name, value)
    6929
    6930          def is_id(colname):
    6931              if colname in self:
    6932                  return self[colname].type == 'id'
    6933              else:
    6934                  return False
    6935
    6936          for line in reader:
    6937              if not line:
    6938                  break
    6939              if not colnames:
    6940                  colnames = [x.split('.',1)[-1] for x in line][:len(line)]
    6941                  cols, cid = [], []
    6942                  for i,colname in enumerate(colnames):
    6943                      if is_id(colname):
    6944                          cid = i
    6945                      else:
    6946                          cols.append(i)
    6947                      if colname == unique:
    6948                          unique_idx = i
    6949              else:
    6950                  items = [fix(self[colnames[i]], line[i], id_map) \
    6951                               for i in cols if colnames[i] in self.fields]
    6952                  # Validation. Check for duplicate of 'unique' &,
    6953                  # if present, update instead of insert.
    6954                  if not unique or unique not in colnames:
    6955                      new_id = self.insert(**dict(items))
    6956                  else:
    6957                      unique_value = line[unique_idx]
    6958                      query = self._db[self][unique] == unique_value
    6959                      record = self._db(query).select().first()
    6960                      if record:
    6961                          record.update_record(**dict(items))
    6962                          new_id = record[self._id.name]
    6963                      else:
    6964                          new_id = self.insert(**dict(items))
    6965                  if id_map and cid != []:
    6966                      id_map_self[int(line[cid])] = new_id
    6967
    6968 -    def with_alias(self, alias):
    6969          return self._db._adapter.alias(self,alias)
    6970
    6971 -    def on(self, query):
    6972          return Expression(self._db,self._db._adapter.ON,self,query)
    6973
    6974
    6975
    6976 -class Expression(object):
    6977
    6978 -    def __init__(
    6979          self,
    6980          db,
    6981          op,
    6982          first=None,
    6983          second=None,
    6984          type=None,
    6985          ):
    6986
    6987          self.db = db
    6988          self.op = op
    6989          self.first = first
    6990          self.second = second
    6991          ### self._tablename =  first._tablename ## CHECK
    6992          if not type and first and hasattr(first,'type'):
    6993              self.type = first.type
    6994          else:
    6995              self.type = type
    6996
    6997 -    def sum(self):
    6998          return Expression(self.db, self.db._adapter.AGGREGATE, self, 'SUM', self.type)
    6999
    7000 -    def max(self):
    7001          return Expression(self.db, self.db._adapter.AGGREGATE, self, 'MAX', self.type)
    7002
    7003 -    def min(self):
    7004          return Expression(self.db, self.db._adapter.AGGREGATE, self, 'MIN', self.type)
    7005
    7006 -    def len(self):
    7007          return Expression(self.db, self.db._adapter.AGGREGATE, self, 'LENGTH', 'integer')
    7008
    7009 -    def lower(self):
    7010          return Expression(self.db, self.db._adapter.LOWER, self, None, self.type)
    7011
    7012 -    def upper(self):
    7013          return Expression(self.db, self.db._adapter.UPPER, self, None, self.type)
    7014
    7015 -    def year(self):
    7016          return Expression(self.db, self.db._adapter.EXTRACT, self, 'year', 'integer')
    7017
    7018 -    def month(self):
    7019          return Expression(self.db, self.db._adapter.EXTRACT, self, 'month', 'integer')
    7020
    7021 -    def day(self):
    7022          return Expression(self.db, self.db._adapter.EXTRACT, self, 'day', 'integer')
    7023
    7024 -    def hour(self):
    7025          return Expression(self.db, self.db._adapter.EXTRACT, self, 'hour', 'integer')
    7026
    7027 -    def minutes(self):
    7028          return Expression(self.db, self.db._adapter.EXTRACT, self, 'minute', 'integer')
    7029
    7030 -    def coalesce(self,*others):
    7031          return Expression(self.db, self.db._adapter.COALESCE, self, others, self.type)
    7032
    7033 -    def coalesce_zero(self):
    7034          return Expression(self.db, self.db._adapter.COALESCE_ZERO, self, None, self.type)
    7035
    7036 -    def seconds(self):
    7037          return Expression(self.db, self.db._adapter.EXTRACT, self, 'second', 'integer')
    7038
    7039 -    def __getslice__(self, start, stop):
    7040          if start < 0:
    7041              pos0 = '(%s - %d)' % (self.len(), abs(start) - 1)
    7042          else:
    7043              pos0 = start + 1
    7044
    7045          if stop < 0:
    7046              length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0)
    7047          elif stop == sys.maxint:
    7048              length = self.len()
    7049          else:
    7050              length = '(%s - %s)' % (stop + 1, pos0)
    7051          return Expression(self.db,self.db._adapter.SUBSTRING,
    7052                            self, (pos0, length), self.type)
    7053
    7054 -    def __getitem__(self, i):
    7055          return self[i:i + 1]
    7056
    7057 -    def __str__(self):
    7058          return self.db._adapter.expand(self,self.type)
    7059
    7060 -    def __or__(self, other):  # for use in sortby
    7061          return Expression(self.db,self.db._adapter.COMMA,self,other,self.type)
    7062
    7063 -    def __invert__(self):
    7064          if hasattr(self,'_op') and self.op == self.db._adapter.INVERT:
    7065              return self.first
    7066          return Expression(self.db,self.db._adapter.INVERT,self,type=self.type)
    7067
    7068 -    def __add__(self, other):
    7069          return Expression(self.db,self.db._adapter.ADD,self,other,self.type)
    7070
    7071 -    def __sub__(self, other):
    7072          if self.type == 'integer':
    7073              result_type = 'integer'
    7074          elif self.type in ['date','time','datetime','double']:
    7075              result_type = 'double'
    7076          else:
    7077              raise SyntaxError, "subtraction operation not supported for type"
    7078          return Expression(self.db,self.db._adapter.SUB,self,other,
    7079                            result_type)
    7080 -    def __mul__(self, other):
    7081          return Expression(self.db,self.db._adapter.MUL,self,other,self.type)
    7082
    7083 -    def __div__(self, other):
    7084          return Expression(self.db,self.db._adapter.DIV,self,other,self.type)
    7085
    7086 -    def __mod__(self, other):
    7087          return Expression(self.db,self.db._adapter.MOD,self,other,self.type)
    7088
    7089 -    def __eq__(self, value):
    7090          return Query(self.db, self.db._adapter.EQ, self, value)
    7091
    7092 -    def __ne__(self, value):
    7093          return Query(self.db, self.db._adapter.NE, self, value)
    7094
    7095 -    def __lt__(self, value):
    7096          return Query(self.db, self.db._adapter.LT, self, value)
    7097
    7098 -    def __le__(self, value):
    7099          return Query(self.db, self.db._adapter.LE, self, value)
    7100
    7101 -    def __gt__(self, value):
    7102          return Query(self.db, self.db._adapter.GT, self, value)
    7103
    7104 -    def __ge__(self, value):
    7105          return Query(self.db, self.db._adapter.GE, self, value)
    7106
    7107 -    def like(self, value, case_sensitive=False):
    7108          op = case_sensitive and self.db._adapter.LIKE or self.db._adapter.ILIKE
    7109          return Query(self.db, op, self, value)
    7110
    7111 -    def regexp(self, value):
    7112          return Query(self.db, self.db._adapter.REGEXP, self, value)
    7113
    7114 -    def belongs(self, value):
    7115          if isinstance(value,Query):
    7116              value = self.db(value)._select(value.first._table._id)
    7117          return Query(self.db, self.db._adapter.BELONGS, self, value)
    7118
    7119 -    def startswith(self, value):
    7120          if not self.type in ('string', 'text'):
    7121              raise SyntaxError, "startswith used with incompatible field type"
    7122          return Query(self.db, self.db._adapter.STARTSWITH, self, value)
    7123
    7124 -    def endswith(self, value):
    7125          if not self.type in ('string', 'text'):
    7126              raise SyntaxError, "endswith used with incompatible field type"
    7127          return Query(self.db, self.db._adapter.ENDSWITH, self, value)
    7128
    7129 -    def contains(self, value, all=False):
    7130          if isinstance(value,(list, tuple)):
    7131              subqueries = [self.contains(str(v).strip()) for v in value if str(v).strip()]
    7132              return reduce(all and AND or OR, subqueries)
    7133          if not self.type in ('string', 'text') and not self.type.startswith('list:'):
    7134              raise SyntaxError, "contains used with incompatible field type"
    7135          return Query(self.db, self.db._adapter.CONTAINS, self, value)
    7136
    7137 -    def with_alias(self, alias):
    7138          return Expression(self.db, self.db._adapter.AS, self, alias, self.type)
    7139
    7140      # for use in both Query and sortby
    7141
    7142
    7143 -class SQLCustomType(object):
    7144      """
    7145      allows defining of custom SQL types
    7146
    7147      Example::
    7148
    7149          decimal = SQLCustomType(
    7150              type ='double',
    7151              native ='integer',
    7152              encoder =(lambda x: int(float(x) * 100)),
    7153              decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
    7154              )
    7155
    7156          db.define_table(
    7157              'example',
    7158              Field('value', type=decimal)
    7159              )
    7160
    7161      :param type: the web2py type (default = 'string')
    7162      :param native: the backend type
    7163      :param encoder: how to encode the value to store it in the backend
    7164      :param decoder: how to decode the value retrieved from the backend
    7165      :param validator: what validators to use ( default = None, will use the
    7166          default validator for type)
    7167      """
    7168
    7169 -    def __init__(
    7170          self,
    7171          type='string',
    7172          native=None,
    7173          encoder=None,
    7174          decoder=None,
    7175          validator=None,
    7176          _class=None,
    7177          ):
    7178
    7179          self.type = type
    7180          self.native = native
    7181          self.encoder = encoder or (lambda x: x)
    7182          self.decoder = decoder or (lambda x: x)
    7183          self.validator = validator
    7184          self._class = _class or type
    7185
    7186 -    def startswith(self, text=None):
    7187          try:
    7188              return self.type.startswith(self, text)
    7189          except TypeError:
    7190              return False
    7191
    7192 -    def __getslice__(self, a=0, b=100):
    7193          return None
    7194
    7195 -    def __getitem__(self, i):
    7196          return None
    7197
    7198 -    def __str__(self):
    7199          return self._class
    7200
    7201 -class FieldVirtual(object):
    7202 -    def __init__(self, f):
    7203          self.f = f
    7204
    7205 -class FieldLazy(object):
    7206 -    def __init__(self, f, handler=None):
    7207          self.f = f
    7208          self.handler = handler
    7209
    7210
    7211 -class Field(Expression):
    7212
    7213      Virtual = FieldVirtual
    7214      Lazy = FieldLazy
    7215
    7216      """
    7217      an instance of this class represents a database field
    7218
    7219      example::
    7220
    7221          a = Field(name, 'string', length=32, default=None, required=False,
    7222              requires=IS_NOT_EMPTY(), ondelete='CASCADE',
    7223              notnull=False, unique=False,
    7224              uploadfield=True, widget=None, label=None, comment=None,
    7225              uploadfield=True, # True means store on disk,
    7226                                # 'a_field_name' means store in this field in db
    7227                                # False means file content will be discarded.
    7228              writable=True, readable=True, update=None, authorize=None,
    7229              autodelete=False, represent=None, uploadfolder=None,
    7230              uploadseparate=False # upload to separate directories by uuid_keys
    7231                                   # first 2 character and tablename.fieldname
    7232                                   # False - old behavior
    7233                                   # True - put uploaded file in
    7234                                   #   <uploaddir>/<tablename>.<fieldname>/uuid_key[:2]
    7235                                   #        directory)
    7236
    7237      to be used as argument of DAL.define_table
    7238
    7239      allowed field types:
    7240      string, boolean, integer, double, text, blob,
    7241      date, time, datetime, upload, password
    7242
    7243      strings must have a length of Adapter.maxcharlength by default (512 or 255 for mysql)
    7244      fields should have a default or they will be required in SQLFORMs
    7245      the requires argument is used to validate the field input in SQLFORMs
    7246
    7247      """
    7248
    7249 -    def __init__(
    7250          self,
    7251          fieldname,
    7252          type='string',
    7253          length=None,
    7254          default=DEFAULT,
    7255          required=False,
    7256          requires=DEFAULT,
    7257          ondelete='CASCADE',
    7258          notnull=False,
    7259          unique=False,
    7260          uploadfield=True,
    7261          widget=None,
    7262          label=DEFAULT,
    7263          comment=None,
    7264          writable=True,
    7265          readable=True,
    7266          update=None,
    7267          authorize=None,
    7268          autodelete=False,
    7269          represent=None,
    7270          uploadfolder=None,
    7271          uploadseparate=False,
    7272          compute=None,
    7273          custom_store=None,
    7274          custom_retrieve=None,
    7275          custom_delete=None,
    7276          ):
    7277          self.db = None
    7278          self.op = None
    7279          self.first = None
    7280          self.second = None
    7281          if not isinstance(fieldname,str):
    7282              raise SyntaxError, "missing field name"
    7283          self.name = fieldname = cleanup(fieldname)
    7284          if hasattr(Table,fieldname) or fieldname[0] == '_' or \
    7285                  regex_python_keywords.match(fieldname):
    7286              raise SyntaxError, 'Field: invalid field name: %s' % fieldname
    7287          if isinstance(type, Table):
    7288              type = 'reference ' + type._tablename
    7289          self.type = type  # 'string', 'integer'
    7290          self.length = (length is None) and DEFAULTLENGTH.get(type,512) or length
    7291          if default is DEFAULT:
    7292              self.default = update or None
    7293          else:
    7294              self.default = default
    7295          self.required = required  # is this field required
    7296          self.ondelete = ondelete.upper()  # this is for reference fields only
    7297          self.notnull = notnull
    7298          self.unique = unique
    7299          self.uploadfield = uploadfield
    7300          self.uploadfolder = uploadfolder
    7301          self.uploadseparate = uploadseparate
    7302          self.widget = widget
    7303          if label is DEFAULT:
    7304              self.label = fieldname.replace('_', ' ').title()
    7305          else:
    7306              self.label = label or ''
    7307          self.comment = comment
    7308          self.writable = writable
    7309          self.readable = readable
    7310          self.update = update
    7311          self.authorize = authorize
    7312          self.autodelete = autodelete
    7313          if not represent and type in ('list:integer','list:string'):
    7314              represent=lambda x,r=None: ', '.join(str(y) for y in x or [])
    7315          self.represent = represent
    7316          self.compute = compute
    7317          self.isattachment = True
    7318          self.custom_store = custom_store
    7319          self.custom_retrieve = custom_retrieve
    7320          self.custom_delete = custom_delete
    7321          if self.label is None:
    7322              self.label = fieldname.replace('_',' ').title()
    7323          if requires is None:
    7324              self.requires = []
    7325          else:
    7326              self.requires = requires
    7327
    7328 -    def store(self, file, filename=None, path=None):
    7329          if self.custom_store:
    7330              return self.custom_store(file,filename,path)
    7331          if isinstance(file, cgi.FieldStorage):
    7332              file = file.file
    7333              filename = filename or file.filename
    7334          elif not filename:
    7335              filename = file.name
    7336          filename = os.path.basename(filename.replace('/', os.sep)\
    7337                                          .replace('\\', os.sep))
    7338          m = re.compile('\.(?P<e>\w{1,5})$').search(filename)
    7339          extension = m and m.group('e') or 'txt'
    7340          uuid_key = web2py_uuid().replace('-', '')[-16:]
    7341          encoded_filename = base64.b16encode(filename).lower()
    7342          newfilename = '%s.%s.%s.%s' % \
    7343              (self._tablename, self.name, uuid_key, encoded_filename)
    7344          newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension
    7345          if isinstance(self.uploadfield,Field):
    7346              blob_uploadfield_name = self.uploadfield.uploadfield
    7347              keys={self.uploadfield.name: newfilename,
    7348                    blob_uploadfield_name: file.read()}
    7349              self.uploadfield.table.insert(**keys)
    7350          elif self.uploadfield == True:
    7351              if path:
    7352                  pass
    7353              elif self.uploadfolder:
    7354                  path = self.uploadfolder
    7355              elif self.db._adapter.folder:
    7356                  path = os.path.join(self.db._adapter.folder, '..', 'uploads')
    7357              else:
    7358                  raise RuntimeError, "you must specify a Field(...,uploadfolder=...)"
    7359              if self.uploadseparate:
    7360                  path = os.path.join(path,"%s.%s" % (self._tablename, self.name),uuid_key[:2])
    7361              if not os.path.exists(path):
    7362                  os.makedirs(path)
    7363              pathfilename = os.path.join(path, newfilename)
    7364              dest_file = open(pathfilename, 'wb')
    7365              try:
    7366                  shutil.copyfileobj(file, dest_file)
    7367              except IOError:
    7368                  raise IOError, 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename
    7369              dest_file.close()
    7370          return newfilename
    7371
    7372 -    def retrieve(self, name, path=None):
    7373          if self.custom_retrieve:
    7374              return self.custom_retrieve(name, path)
    7375          import http
    7376          if self.authorize or isinstance(self.uploadfield, str):
    7377              row = self.db(self == name).select().first()
    7378              if not row:
    7379                  raise http.HTTP(404)
    7380          if self.authorize and not self.authorize(row):
    7381              raise http.HTTP(403)
    7382          try:
    7383              m = regex_content.match(name)
    7384              if not m or not self.isattachment:
    7385                  raise TypeError, 'Can\'t retrieve %s' % name
    7386              filename = base64.b16decode(m.group('name'), True)
    7387              filename = regex_cleanup_fn.sub('_', filename)
    7388          except (TypeError, AttributeError):
    7389              filename = name
    7390          if isinstance(self.uploadfield, str):  # ## if file is in DB
    7391              return (filename, cStringIO.StringIO(row[self.uploadfield] or ''))
    7392          elif isinstance(self.uploadfield,Field):
    7393              blob_uploadfield_name = self.uploadfield.uploadfield
    7394              query = self.uploadfield == name
    7395              data = self.uploadfield.table(query)[blob_uploadfield_name]
    7396              return (filename, cStringIO.StringIO(data))
    7397          else:
    7398              # ## if file is on filesystem
    7399              if path:
    7400                  pass
    7401              elif self.uploadfolder:
    7402                  path = self.uploadfolder
    7403              else:
    7404                  path = os.path.join(self.db._adapter.folder, '..', 'uploads')
    7405              if self.uploadseparate:
    7406                  t = m.group('table')
    7407                  f = m.group('field')
    7408                  u = m.group('uuidkey')
    7409                  path = os.path.join(path,"%s.%s" % (t,f),u[:2])
    7410              return (filename, open(os.path.join(path, name), 'rb'))
    7411
    7412 -    def formatter(self, value):
    7413          if value is None or not self.requires:
    7414              return value
    7415          if not isinstance(self.requires, (list, tuple)):
    7416              requires = [self.requires]
    7417          elif isinstance(self.requires, tuple):
    7418              requires = list(self.requires)
    7419          else:
    7420              requires = copy.copy(self.requires)
    7421          requires.reverse()
    7422          for item in requires:
    7423              if hasattr(item, 'formatter'):
    7424                  value = item.formatter(value)
    7425          return value
    7426
    7427 -    def validate(self, value):
    7428          if not self.requires:
    7429              return (value, None)
    7430          requires = self.requires
    7431          if not isinstance(requires, (list, tuple)):
    7432              requires = [requires]
    7433          for validator in requires:
    7434              (value, error) = validator(value)
    7435              if error:
    7436                  return (value, error)
    7437          return (value, None)
    7438
    7439 -    def count(self):
    7440          return Expression(self.db, self.db._adapter.AGGREGATE, self, 'COUNT', 'integer')
    7441
    7442 -    def __nonzero__(self):
    7443          return True
    7444
    7445 -    def __str__(self):
    7446          try:
    7447              return '%s.%s' % (self.tablename, self.name)
    7448          except:
    7449              return '<no table>.%s' % self.name
    7450
    7451
    7452 -def raw(s): return Expression(None,s)
    7453
    7454 -class Query(object):
    7455
    7456      """
    7457      a query object necessary to define a set.
    7458      it can be stored or can be passed to DAL.__call__() to obtain a Set
    7459
    7460      Example::
    7461
    7462          query = db.users.name=='Max'
    7463          set = db(query)
    7464          records = set.select()
    7465
    7466      """
    7467
    7468 -    def __init__(
    7469          self,
    7470          db,
    7471          op,
    7472          first=None,
    7473          second=None,
    7474          ignore_common_filters = False,
    7475          ):
    7476          self.db = self._db = db
    7477          self.op = op
    7478          self.first = first
    7479          self.second = second
    7480          self.ignore_common_filters = ignore_common_filters
    7481
    7482 -    def __str__(self):
    7483          return self.db._adapter.expand(self)
    7484
    7485 -    def __and__(self, other):
    7486          return Query(self.db,self.db._adapter.AND,self,other)
    7487
    7488 -    def __or__(self, other):
    7489          return Query(self.db,self.db._adapter.OR,self,other)
    7490
    7491 -    def __invert__(self):
    7492          if self.op==self.db._adapter.NOT:
    7493              return self.first
    7494          return Query(self.db,self.db._adapter.NOT,self)
    7495
    7496
    7497  regex_quotes = re.compile("'[^']*'")
    7498
    7499
    7500 -def xorify(orderby):
    7501      if not orderby:
    7502          return None
    7503      orderby2 = orderby[0]
    7504      for item in orderby[1:]:
    7505          orderby2 = orderby2 | item
    7506      return orderby2
    7507
    7508 -def use_common_filters(query):
    7509      return (query and hasattr(query,'ignore_common_filters') and \
    7510                  not query.ignore_common_filters)
    7511
    7512 -class Set(object):
    7513
    7514      """
    7515      a Set represents a set of records in the database,
    7516      the records are identified by the query=Query(...) object.
    7517      normally the Set is generated by DAL.__call__(Query(...))
    7518
    7519      given a set, for example
    7520         set = db(db.users.name=='Max')
    7521      you can:
    7522         set.update(db.users.name='Massimo')
    7523         set.delete() # all elements in the set
    7524         set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10))
    7525      and take subsets:
    7526         subset = set(db.users.id<5)
    7527      """
    7528
    7529 -    def __init__(self, db, query, ignore_common_filters = None):
    7530          self.db = db
    7531          self._db = db # for backward compatibility
    7532          if not ignore_common_filters is None and \
    7533                  use_common_filters(query) == ignore_common_filters:
    7534              query = copy.copy(query)
    7535              query.ignore_common_filters = ignore_common_filters
    7536          self.query = query
    7537
    7538 -    def __call__(self, query, ignore_common_filters=False):
    7539          if isinstance(query,Table):
    7540              query = query._id>0
    7541          elif isinstance(query,str):
    7542              query = raw(query)
    7543          elif isinstance(query,Field):
    7544              query = query!=None
    7545          if self.query:
    7546              return Set(self.db, self.query & query,
    7547                         ignore_common_filters = ignore_common_filters)
    7548          else:
    7549              return Set(self.db, query,
    7550                         ignore_common_filters = ignore_common_filters)
    7551
    7552 -    def _count(self,distinct=None):
    7553          return self.db._adapter._count(self.query,distinct)
    7554
    7555 -    def _select(self, *fields, **attributes):
    7556          adapter = self.db._adapter
    7557          fields = adapter.expand_all(fields, adapter.tables(self.query))
    7558          return adapter._select(self.query,fields,attributes)
    7559
    7560 -    def _delete(self):
    7561          tablename=self.db._adapter.get_table(self.query)
    7562          return self.db._adapter._delete(tablename,self.query)
    7563
    7564 -    def _update(self, **update_fields):
    7565          tablename = self.db._adapter.get_table(self.query)
    7566          fields = self.db[tablename]._listify(update_fields,update=True)
    7567          return self.db._adapter._update(tablename,self.query,fields)
    7568
    7569 -    def isempty(self):
    7570          return not self.select(limitby=(0,1))
    7571
    7572 -    def count(self,distinct=None):
    7573          return self.db._adapter.count(self.query,distinct)
    7574
    7575 -    def select(self, *fields, **attributes):
    7576          adapter = self.db._adapter
    7577          fields = adapter.expand_all(fields, adapter.tables(self.query))
    7578          return adapter.select(self.query,fields,attributes)
    7579
    7580 -    def delete(self):
    7581          tablename=self.db._adapter.get_table(self.query)
    7582          self.delete_uploaded_files()
    7583          return self.db._adapter.delete(tablename,self.query)
    7584
    7585 -    def update(self, **update_fields):
    7586          tablename = self.db._adapter.get_table(self.query)
    7587          fields = self.db[tablename]._listify(update_fields,update=True)
    7588          if not fields:
    7589              raise SyntaxError, "No fields to update"
    7590          self.delete_uploaded_files(update_fields)
    7591          return self.db._adapter.update(tablename,self.query,fields)
    7592
    7593 -    def validate_and_update(self, **update_fields):
    7594          tablename = self.db._adapter.get_table(self.query)
    7595          response = Row()
    7596          response.errors = Row()
    7597          new_fields = copy.copy(update_fields)
    7598          for key,value in update_fields.items():
    7599              value,error = self.db[tablename][key].validate(value)
    7600              if error:
    7601                  response.errors[key] = error
    7602              else:
    7603                  new_fields[key] = value
    7604          fields = self.db[tablename]._listify(new_fields,update=True)
    7605          if not fields:
    7606              raise SyntaxError, "No fields to update"
    7607          if response.errors:
    7608              response.updated = None
    7609          else:
    7610              self.delete_uploaded_files(new_fields)
    7611              response.updated = self.db._adapter.update(tablename,self.query,fields)
    7612          return response
    7613
    7614 -    def delete_uploaded_files(self, upload_fields=None):
    7615          table = self.db[self.db._adapter.tables(self.query)[0]]
    7616          # ## mind uploadfield==True means file is not in DB
    7617          if upload_fields:
    7618              fields = upload_fields.keys()
    7619          else:
    7620              fields = table.fields
    7621          fields = [f for f in fields if table[f].type == 'upload'
    7622                     and table[f].uploadfield == True
    7623                     and table[f].autodelete]
    7624          if not fields:
    7625              return
    7626          for record in self.select(*[table[f] for f in fields]):
    7627              for fieldname in fields:
    7628                  field = table[fieldname]
    7629                  oldname = record.get(fieldname, None)
    7630                  if not oldname:
    7631                      continue
    7632                  if upload_fields and oldname == upload_fields[fieldname]:
    7633                      continue
    7634                  if field.custom_delete:
    7635                      field.custom_delete(oldname)
    7636                  else:
    7637                      uploadfolder = field.uploadfolder
    7638                      if not uploadfolder:
    7639                          uploadfolder = os.path.join(self.db._adapter.folder, '..', 'uploads')
    7640                      if field.uploadseparate:
    7641                          items = oldname.split('.')
    7642                          uploadfolder = os.path.join(uploadfolder,
    7643                                                      "%s.%s" % (items[0], items[1]),
    7644                                                      items[2][:2])
    7645                      oldpath = os.path.join(uploadfolder, oldname)
    7646                      if os.path.exists(oldpath):
    7647                          os.unlink(oldpath)
    7648
    7649 -def update_record(pack, a=None):
    7650      (colset, table, id) = pack
    7651      b = a or dict(colset)
    7652      c = dict([(k,v) for (k,v) in b.items() if k in table.fields and table[k].type!='id'])
    7653      table._db(table._id==id).update(**c)
    7654      for (k, v) in c.items():
    7655          colset[k] = v
    7656
    7657 -class VirtualCommand(object):
    7658 -    def __init__(self,method,row):
    7659          self.method=method
    7660          self.row=row
    7661 -    def __call__(self,*args,**kwargs):
    7662          return self.method(self.row,*args,**kwargs)
    7663
    7664 -def lazy_virtualfield(f):
    7665      f.__lazy__ = True
    7666      return f
    7667
    7668 -class Rows(object):
    7669
    7670      """
    7671      A wrapper for the return value of a select. It basically represents a table.
    7672      It has an iterator and each row is represented as a dictionary.
    7673      """
    7674
    7675      # ## TODO: this class still needs some work to care for ID/OID
    7676
    7677 -    def __init__(
    7678          self,
    7679          db=None,
    7680          records=[],
    7681          colnames=[],
    7682          compact=True,
    7683          rawrows=None
    7684          ):
    7685          self.db = db
    7686          self.records = records
    7687          self.colnames = colnames
    7688          self.compact = compact
    7689          self.response = rawrows
    7690
    7691 -    def setvirtualfields(self,**keyed_virtualfields):
    7692          """
    7693          db.define_table('x',Field('number','integer'))
    7694          if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)]
    7695
    7696          from gluon.dal import lazy_virtualfield
    7697
    7698          class MyVirtualFields(object):
    7699              # normal virtual field (backward compatible, discouraged)
    7700              def normal_shift(self): return self.x.number+1
    7701              # lazy virtual field (because of @staticmethod)
    7702              @lazy_virtualfield
    7703              def lazy_shift(instance,row,delta=4): return row.x.number+delta
    7704          db.x.virtualfields.append(MyVirtualFields())
    7705
    7706          for row in db(db.x).select():
    7707              print row.number, row.normal_shift, row.lazy_shift(delta=7)
    7708          """
    7709          if not keyed_virtualfields:
    7710              return self
    7711          for row in self.records:
    7712              for (tablename,virtualfields) in keyed_virtualfields.items():
    7713                  attributes = dir(virtualfields)
    7714                  if not tablename in row:
    7715                      box = row[tablename] = Row()
    7716                  else:
    7717                      box = row[tablename]
    7718                  updated = False
    7719                  for attribute in attributes:
    7720                      if attribute[0] != '_':
    7721                          method = getattr(virtualfields,attribute)
    7722                          if hasattr(method,'__lazy__'):
    7723                              box[attribute]=VirtualCommand(method,row)
    7724                          elif type(method)==types.MethodType:
    7725                              if not updated:
    7726                                  virtualfields.__dict__.update(row)
    7727                                  updated = True
    7728                              box[attribute]=method()
    7729          return self
    7730
    7731 -    def __and__(self,other):
    7732          if self.colnames!=other.colnames: raise Exception, 'Cannot & incompatible Rows objects'
    7733          records = self.records+other.records
    7734          return Rows(self.db,records,self.colnames)
    7735
    7736 -    def __or__(self,other):
    7737          if self.colnames!=other.colnames: raise Exception, 'Cannot | incompatible Rows objects'
    7738          records = self.records
    7739          records += [record for record in other.records \
    7740                          if not record in records]
    7741          return Rows(self.db,records,self.colnames)
    7742
    7743 -    def __nonzero__(self):
    7744          if len(self.records):
    7745              return 1
    7746          return 0
    7747
    7748 -    def __len__(self):
    7749          return len(self.records)
    7750
    7751 -    def __getslice__(self, a, b):
    7752          return Rows(self.db,self.records[a:b],self.colnames)
    7753
    7754 -    def __getitem__(self, i):
    7755          row = self.records[i]
    7756          keys = row.keys()
    7757          if self.compact and len(keys) == 1 and keys[0] != '_extra':
    7758              return row[row.keys()[0]]
    7759          return row
    7760
    7761 -    def __iter__(self):
    7762          """
    7763          iterator over records
    7764          """
    7765
    7766          for i in xrange(len(self)):
    7767              yield self[i]
    7768
    7769 -    def __str__(self):
    7770          """
    7771          serializes the table into a csv file
    7772          """
    7773
    7774          s = cStringIO.StringIO()
    7775          self.export_to_csv_file(s)
    7776          return s.getvalue()
    7777
    7778 -    def first(self):
    7779          if not self.records:
    7780              return None
    7781          return self[0]
    7782
    7783 -    def last(self):
    7784          if not self.records:
    7785              return None
    7786          return self[-1]
    7787
    7788 -    def find(self,f):
    7789          """
    7790          returns a new Rows object, a subset of the original object,
    7791          filtered by the function f
    7792          """
    7793          if not self.records:
    7794              return Rows(self.db, [], self.colnames)
    7795          records = []
    7796          for i in range(0,len(self)):
    7797              row = self[i]
    7798              if f(row):
    7799                  records.append(self.records[i])
    7800          return Rows(self.db, records, self.colnames)
    7801
    7802 -    def exclude(self, f):
    7803          """
    7804          removes elements from the calling Rows object, filtered by the function f,
    7805          and returns a new Rows object containing the removed elements
    7806          """
    7807          if not self.records:
    7808              return Rows(self.db, [], self.colnames)
    7809          removed = []
    7810          i=0
    7811          while i<len(self):
    7812              row = self[i]
    7813              if f(row):
    7814                  removed.append(self.records[i])
    7815                  del self.records[i]
    7816              else:
    7817                  i += 1
    7818          return Rows(self.db, removed, self.colnames)
    7819
    7820 -    def sort(self, f, reverse=False):
    7821          """
    7822          returns a list of sorted elements (not sorted in place)
    7823          """
    7824          return Rows(self.db,sorted(self,key=f,reverse=reverse),self.colnames)
    7825
    7826 -    def group_by_value(self, field):
    7827          """
    7828          regroups the rows, by one of the fields
    7829          """
    7830          if not self.records:
    7831              return {}
    7832          key = str(field)
    7833          grouped_row_group = dict()
    7834
    7835          for row in self:
    7836              value = row[key]
    7837              if not value in grouped_row_group:
    7838                  grouped_row_group[value] = [row]
    7839              else:
    7840                  grouped_row_group[value].append(row)
    7841          return grouped_row_group
    7842
    7843 -    def as_list(self,
    7844                  compact=True,
    7845                  storage_to_dict=True,
    7846                  datetime_to_str=True):
    7847          """
    7848          returns the data as a list or dictionary.
    7849          :param storage_to_dict: when True returns a dict, otherwise a list(default True)
    7850          :param datetime_to_str: convert datetime fields as strings (default True)
    7851          """
    7852          (oc, self.compact) = (self.compact, compact)
    7853          if storage_to_dict:
    7854              items = [item.as_dict(datetime_to_str) for item in self]
    7855          else:
    7856              items = [item for item in self]
    7857          self.compact = compact
    7858          return items
    7859
    7860
    7861 -    def as_dict(self,
    7862                  key='id',
    7863                  compact=True,
    7864                  storage_to_dict=True,
    7865                  datetime_to_str=True):
    7866          """
    7867          returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False)
    7868
    7869          :param key: the name of the field to be used as dict key, normally the id
    7870          :param compact: ? (default True)
    7871          :param storage_to_dict: when True returns a dict, otherwise a list(default True)
    7872          :param datetime_to_str: convert datetime fields as strings (default True)
    7873          """
    7874          rows = self.as_list(compact, storage_to_dict, datetime_to_str)
    7875          if isinstance(key,str) and key.count('.')==1:
    7876              (table, field) = key.split('.')
    7877              return dict([(r[table][field],r) for r in rows])
    7878          elif isinstance(key,str):
    7879              return dict([(r[key],r) for r in rows])
    7880          else:
    7881              return dict([(key(r),r) for r in rows])
    7882
    7883 -    def export_to_csv_file(self, ofile, null='<NULL>', *args, **kwargs):
    7884          """
    7885          export data to csv, the first line contains the column names
    7886
    7887          :param ofile: where the csv must be exported to
    7888          :param null: how null values must be represented (default '<NULL>')
    7889          :param delimiter: delimiter to separate values (default ',')
    7890          :param quotechar: character to use to quote string values (default '"')
    7891          :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL)
    7892          :param represent: use the fields .represent value (default False)
    7893          :param colnames: list of column names to use (default self.colnames)
    7894                           This will only work when exporting rows objects!!!!
    7895                           DO NOT use this with db.export_to_csv()
    7896          """
    7897          delimiter = kwargs.get('delimiter', ',')
    7898          quotechar = kwargs.get('quotechar', '"')
    7899          quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
    7900          represent = kwargs.get('represent', False)
    7901          writer = csv.writer(ofile, delimiter=delimiter,
    7902                              quotechar=quotechar, quoting=quoting)
    7903          colnames = kwargs.get('colnames', self.colnames)
    7904          write_colnames = kwargs.get('write_colnames',True)
    7905          # a proper csv starting with the column names
    7906          if write_colnames:
    7907              writer.writerow(colnames)
    7908
    7909          def none_exception(value):
    7910              """
    7911              returns a cleaned up value that can be used for csv export:
    7912              - unicode text is encoded as such
    7913              - None values are replaced with the given representation (default <NULL>)
    7914              """
    7915              if value is None:
    7916                  return null
    7917              elif isinstance(value, unicode):
    7918                  return value.encode('utf8')
    7919              elif isinstance(value,Reference):
    7920                  return int(value)
    7921              elif hasattr(value, 'isoformat'):
    7922                  return value.isoformat()[:19].replace('T', ' ')
    7923              elif isinstance(value, (list,tuple)): # for type='list:..'
    7924                  return bar_encode(value)
    7925              return value
    7926
    7927          for record in self:
    7928              row = []
    7929              for col in colnames:
    7930                  if not regex_table_field.match(col):
    7931                      row.append(record._extra[col])
    7932                  else:
    7933                      (t, f) = col.split('.')
    7934                      field = self.db[t][f]
    7935                      if isinstance(record.get(t, None), (Row,dict)):
    7936                          value = record[t][f]
    7937                      else:
    7938                          value = record[f]
    7939                      if field.type=='blob' and not value is None:
    7940                          value = base64.b64encode(value)
    7941                      elif represent and field.represent:
    7942                          value = field.represent(value)
    7943                      row.append(none_exception(value))
    7944              writer.writerow(row)
    7945
    7946 -    def xml(self):
    7947          """
    7948          serializes the table using sqlhtml.SQLTABLE (if present)
    7949          """
    7950
    7951          import sqlhtml
    7952          return sqlhtml.SQLTABLE(self).xml()
    7953
    7954 -    def json(self, mode='object', default=None):
    7955          """
    7956          serializes the table to a JSON list of objects
    7957          """
    7958          mode = mode.lower()
    7959          if not mode in ['object', 'array']:
    7960              raise SyntaxError, 'Invalid JSON serialization mode: %s' % mode
    7961
    7962          def inner_loop(record, col):
    7963              (t, f) = col.split('.')
    7964              res = None
    7965              if not regex_table_field.match(col):
    7966                  key = col
    7967                  res = record._extra[col]
    7968              else:
    7969                  key = f
    7970                  if isinstance(record.get(t, None), Row):
    7971                      res = record[t][f]
    7972                  else:
    7973                      res = record[f]
    7974              if mode == 'object':
    7975                  return (key, res)
    7976              else:
    7977                  return res
    7978
    7979          if mode == 'object':
    7980              items = [dict([inner_loop(record, col) for col in
    7981                       self.colnames]) for record in self]
    7982          else:
    7983              items = [[inner_loop(record, col) for col in self.colnames]
    7984                       for record in self]
    7985          if have_serializers:
    7986              return serializers.json(items,default=default or serializers.custom_json)
    7987          else:
    7988              import simplejson
    7989              return simplejson.dumps(items)
    7990
    7991 -def Rows_unpickler(data):
    7992      return cPickle.loads(data)
    7993
    7994 -def Rows_pickler(data):
    7995      return Rows_unpickler, \
    7996          (cPickle.dumps(data.as_list(storage_to_dict=False,
    7997                                      datetime_to_str=False)),)
    7998
    7999  copy_reg.pickle(Rows, Rows_pickler, Rows_unpickler)
    8000
    8001
    8002  ################################################################################
    8003  # dummy function used to define some doctests
    8004  ################################################################################
    8005
    8006 -def test_all():
    8007      """
    8008
    8009      >>> if len(sys.argv)<2: db = DAL(\"sqlite://test.db\")
    8010      >>> if len(sys.argv)>1: db = DAL(sys.argv[1])
    8011      >>> tmp = db.define_table('users',\
    8012                Field('stringf', 'string', length=32, required=True),\
    8013                Field('booleanf', 'boolean', default=False),\
    8014                Field('passwordf', 'password', notnull=True),\
    8015                Field('uploadf', 'upload'),\
    8016                Field('blobf', 'blob'),\
    8017                Field('integerf', 'integer', unique=True),\
    8018                Field('doublef', 'double', unique=True,notnull=True),\
    8019                Field('datef', 'date', default=datetime.date.today()),\
    8020                Field('timef', 'time'),\
    8021                Field('datetimef', 'datetime'),\
    8022                migrate='test_user.table')
    8023
    8024     Insert a field
    8025
    8026      >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\
    8027                         uploadf=None, integerf=5, doublef=3.14,\
    8028                         datef=datetime.date(2001, 1, 1),\
    8029                         timef=datetime.time(12, 30, 15),\
    8030                         datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15))
    8031      1
    8032
    8033      Drop the table
    8034
    8035      >>> db.users.drop()
    8036
    8037      Examples of insert, select, update, delete
    8038
    8039      >>> tmp = db.define_table('person',\
    8040                Field('name'),\
    8041                Field('birth','date'),\
    8042                migrate='test_person.table')
    8043      >>> person_id = db.person.insert(name=\"Marco\",birth='2005-06-22')
    8044      >>> person_id = db.person.insert(name=\"Massimo\",birth='1971-12-21')
    8045
    8046      commented len(db().select(db.person.ALL))
    8047      commented 2
    8048
    8049      >>> me = db(db.person.id==person_id).select()[0] # test select
    8050      >>> me.name
    8051      'Massimo'
    8052      >>> db(db.person.name=='Massimo').update(name='massimo') # test update
    8053      1
    8054      >>> db(db.person.name=='Marco').select().first().delete_record() # test delete
    8055      1
    8056
    8057      Update a single record
    8058
    8059      >>> me.update_record(name=\"Max\")
    8060      >>> me.name
    8061      'Max'
    8062
    8063      Examples of complex search conditions
    8064
    8065      >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select())
    8066      1
    8067      >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select())
    8068      1
    8069      >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select())
    8070      1
    8071      >>> me = db(db.person.id==person_id).select(db.person.name)[0]
    8072      >>> me.name
    8073      'Max'
    8074
    8075      Examples of search conditions using extract from date/datetime/time
    8076
    8077      >>> len(db(db.person.birth.month()==12).select())
    8078      1
    8079      >>> len(db(db.person.birth.year()>1900).select())
    8080      1
    8081
    8082      Example of usage of NULL
    8083
    8084      >>> len(db(db.person.birth==None).select()) ### test NULL
    8085      0
    8086      >>> len(db(db.person.birth!=None).select()) ### test NULL
    8087      1
    8088
    8089      Examples of search conditions using lower, upper, and like
    8090
    8091      >>> len(db(db.person.name.upper()=='MAX').select())
    8092      1
    8093      >>> len(db(db.person.name.like('%ax')).select())
    8094      1
    8095      >>> len(db(db.person.name.upper().like('%AX')).select())
    8096      1
    8097      >>> len(db(~db.person.name.upper().like('%AX')).select())
    8098      0
    8099
    8100      orderby, groupby and limitby
    8101
    8102      >>> people = db().select(db.person.name, orderby=db.person.name)
    8103      >>> order = db.person.name|~db.person.birth
    8104      >>> people = db().select(db.person.name, orderby=order)
    8105
    8106      >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name)
    8107
    8108      >>> people = db().select(db.person.name, orderby=order, limitby=(0,100))
    8109
    8110      Example of one 2 many relation
    8111
    8112      >>> tmp = db.define_table('dog',\
    8113                 Field('name'),\
    8114                 Field('birth','date'),\
    8115                 Field('owner',db.person),\
    8116                 migrate='test_dog.table')
    8117      >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id)
    8118      1
    8119
    8120      A simple JOIN
    8121
    8122      >>> len(db(db.dog.owner==db.person.id).select())
    8123      1
    8124
    8125      >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id)))
    8126      1
    8127
    8128      Drop tables
    8129
    8130      >>> db.dog.drop()
    8131      >>> db.person.drop()
    8132
    8133      Example of many 2 many relation and Set
    8134
    8135      >>> tmp = db.define_table('author', Field('name'),\
    8136                              migrate='test_author.table')
    8137      >>> tmp = db.define_table('paper', Field('title'),\
    8138                              migrate='test_paper.table')
    8139      >>> tmp = db.define_table('authorship',\
    8140              Field('author_id', db.author),\
    8141              Field('paper_id', db.paper),\
    8142              migrate='test_authorship.table')
    8143      >>> aid = db.author.insert(name='Massimo')
    8144      >>> pid = db.paper.insert(title='QCD')
    8145      >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid)
    8146
    8147      Define a Set
    8148
    8149      >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id))
    8150      >>> rows = authored_papers.select(db.author.name, db.paper.title)
    8151      >>> for row in rows: print row.author.name, row.paper.title
    8152      Massimo QCD
    8153
    8154      Example of search condition using  belongs
    8155
    8156      >>> set = (1, 2, 3)
    8157      >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL)
    8158      >>> print rows[0].title
    8159      QCD
    8160
    8161      Example of search condition using nested select
    8162
    8163      >>> nested_select = db()._select(db.authorship.paper_id)
    8164      >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL)
    8165      >>> print rows[0].title
    8166      QCD
    8167
    8168      Example of expressions
    8169
    8170      >>> mynumber = db.define_table('mynumber', Field('x', 'integer'))
    8171      >>> db(mynumber.id>0).delete()
    8172      0
    8173      >>> for i in range(10): tmp = mynumber.insert(x=i)
    8174      >>> db(mynumber.id>0).select(mynumber.x.sum())[0](mynumber.x.sum())
    8175      45
    8176
    8177      >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2)
    8178      5
    8179
    8180      Output in csv
    8181
    8182      >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip()
    8183      author.name,paper.title\r
    8184      Massimo,QCD
    8185
    8186      Delete all leftover tables
    8187
    8188      >>> DAL.distributed_transaction_commit(db)
    8189
    8190      >>> db.mynumber.drop()
    8191      >>> db.authorship.drop()
    8192      >>> db.author.drop()
    8193      >>> db.paper.drop()
    8194      """
    8195  ################################################################################
    8196  # deprecated since the new DAL; here only for backward compatibility
    8197  ################################################################################
    8198
    8199  SQLField = Field
    8200  SQLTable = Table
    8201  SQLXorable = Expression
    8202  SQLQuery = Query
    8203  SQLSet = Set
    8204  SQLRows = Rows
    8205  SQLStorage = Row
    8206  SQLDB = DAL
    8207  GQLDB = DAL
    8208  DAL.Field = Field  # was necessary in gluon/globals.py session.connect
    8209  DAL.Table = Table  # was necessary in gluon/globals.py session.connect
    8210
    8211  ################################################################################
    8212  # run tests
    8213  ################################################################################
    8214
    8215  if __name__ == '__main__':
    8216      import doctest
    8217      doctest.testmod()
    8218