googleapiclient.discovery

Package googleapiclient :: Module discovery |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.discovery

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Client for discovery based APIs.
      16
      17  A client library for Google's discovery based APIs.
      18  """
      19  from __future__ import absolute_import
      20  import six
      21  from six.moves import zip
      22
      23  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      24  __all__ = [
      25      'build',
      26      'build_from_document',
      27      'fix_method_name',
      28      'key2param',
      29      ]
      30
      31  from six import BytesIO
      32  from six.moves import http_client
      33  from six.moves.urllib.parse import urlencode, urlparse, urljoin, \
      34    urlunparse, parse_qsl
      35
      36  # Standard library imports
      37  import copy
      38  try:
      39    from email.generator import BytesGenerator
      40  except ImportError:
      41    from email.generator import Generator as BytesGenerator
      42  from email.mime.multipart import MIMEMultipart
      43  from email.mime.nonmultipart import MIMENonMultipart
      44  import json
      45  import keyword
      46  import logging
      47  import mimetypes
      48  import os
      49  import re
      50
      51  # Third-party imports
      52  import httplib2
      53  import uritemplate
      54
      55  # Local imports
      56  from googleapiclient import mimeparse
      57  from googleapiclient.errors import HttpError
      58  from googleapiclient.errors import InvalidJsonError
      59  from googleapiclient.errors import MediaUploadSizeError
      60  from googleapiclient.errors import UnacceptableMimeTypeError
      61  from googleapiclient.errors import UnknownApiNameOrVersion
      62  from googleapiclient.errors import UnknownFileType
      63  from googleapiclient.http import BatchHttpRequest
      64  from googleapiclient.http import HttpMock
      65  from googleapiclient.http import HttpMockSequence
      66  from googleapiclient.http import HttpRequest
      67  from googleapiclient.http import MediaFileUpload
      68  from googleapiclient.http import MediaUpload
      69  from googleapiclient.model import JsonModel
      70  from googleapiclient.model import MediaModel
      71  from googleapiclient.model import RawModel
      72  from googleapiclient.schema import Schemas
      73  from oauth2client.client import GoogleCredentials
      74
      75  # Oauth2client < 3 has the positional helper in 'util', >= 3 has it
      76  # in '_helpers'.
      77  try:
      78    from oauth2client.util import _add_query_parameter
      79    from oauth2client.util import positional
      80  except ImportError:
      81    from oauth2client._helpers import _add_query_parameter
      82    from oauth2client._helpers import positional
      83
      84
      85  # The client library requires a version of httplib2 that supports RETRIES.
      86  httplib2.RETRIES = 1
      87
      88  logger = logging.getLogger(__name__)
      89
      90  URITEMPLATE = re.compile('{[^}]*}')
      91  VARNAME = re.compile('[a-zA-Z0-9_-]+')
      92  DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
      93                   '{api}/{apiVersion}/rest')
      94  V1_DISCOVERY_URI = DISCOVERY_URI
      95  V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?'
      96                      'version={apiVersion}')
      97  DEFAULT_METHOD_DOC = 'A description of how to use this function'
      98  HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH'])
      99  _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40}
     100  BODY_PARAMETER_DEFAULT_VALUE = {
     101      'description': 'The request body.',
     102      'type': 'object',
     103      'required': True,
     104  }
     105  MEDIA_BODY_PARAMETER_DEFAULT_VALUE = {
     106      'description': ('The filename of the media request body, or an instance '
     107                      'of a MediaUpload object.'),
     108      'type': 'string',
     109      'required': False,
     110  }
     111  MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE = {
     112      'description': ('The MIME type of the media request body, or an instance '
     113                      'of a MediaUpload object.'),
     114      'type': 'string',
     115      'required': False,
     116  }
     117
     118  # Parameters accepted by the stack, but not visible via discovery.
     119  # TODO(dhermes): Remove 'userip' in 'v2'.
     120  STACK_QUERY_PARAMETERS = frozenset(['trace', 'pp', 'userip', 'strict'])
     121  STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'}
     122
     123  # Library-specific reserved words beyond Python keywords.
     124  RESERVED_WORDS = frozenset(['body'])
     125
     126  # patch _write_lines to avoid munging '\r' into '\n'
     127  # ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 )
     128 -class _BytesGenerator(BytesGenerator):
     129    _write_lines = BytesGenerator.write
     130
     131 -def fix_method_name(name):
     132    """Fix method names to avoid reserved word conflicts.
     133
     134    Args:
     135      name: string, method name.
     136
     137    Returns:
     138      The name with a '_' prefixed if the name is a reserved word.
     139    """
     140    if keyword.iskeyword(name) or name in RESERVED_WORDS:
     141      return name + '_'
     142    else:
     143      return name
     144
     145
     146 -def key2param(key):
     147    """Converts key names into parameter names.
     148
     149    For example, converting "max-results" -> "max_results"
     150
     151    Args:
     152      key: string, the method key name.
     153
     154    Returns:
     155      A safe method name based on the key name.
     156    """
     157    result = []
     158    key = list(key)
     159    if not key[0].isalpha():
     160      result.append('x')
     161    for c in key:
     162      if c.isalnum():
     163        result.append(c)
     164      else:
     165        result.append('_')
     166
     167    return ''.join(result)
     168
     169
     170  @positional(2)
     171 -def build(serviceName,
     172            version,
     173            http=None,
     174            discoveryServiceUrl=DISCOVERY_URI,
     175            developerKey=None,
     176            model=None,
     177            requestBuilder=HttpRequest,
     178            credentials=None,
     179            cache_discovery=True,
     180            cache=None):
     181    """Construct a Resource for interacting with an API.
     182
     183    Construct a Resource object for interacting with an API. The serviceName and
     184    version are the names from the Discovery service.
     185
     186    Args:
     187      serviceName: string, name of the service.
     188      version: string, the version of the service.
     189      http: httplib2.Http, An instance of httplib2.Http or something that acts
     190        like it that HTTP requests will be made through.
     191      discoveryServiceUrl: string, a URI Template that points to the location of
     192        the discovery service. It should have two parameters {api} and
     193        {apiVersion} that when filled in produce an absolute URI to the discovery
     194        document for that service.
     195      developerKey: string, key obtained from
     196        https://code.google.com/apis/console.
     197      model: googleapiclient.Model, converts to and from the wire format.
     198      requestBuilder: googleapiclient.http.HttpRequest, encapsulator for an HTTP
     199        request.
     200      credentials: oauth2client.Credentials, credentials to be used for
     201        authentication.
     202      cache_discovery: Boolean, whether or not to cache the discovery doc.
     203      cache: googleapiclient.discovery_cache.base.CacheBase, an optional
     204        cache object for the discovery documents.
     205
     206    Returns:
     207      A Resource object with methods for interacting with the service.
     208    """
     209    params = {
     210        'api': serviceName,
     211        'apiVersion': version
     212        }
     213
     214    if http is None:
     215      http = httplib2.Http()
     216
     217    for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,):
     218      requested_url = uritemplate.expand(discovery_url, params)
     219
     220      try:
     221        content = _retrieve_discovery_doc(requested_url, http, cache_discovery,
     222                                          cache)
     223        return build_from_document(content, base=discovery_url, http=http,
     224            developerKey=developerKey, model=model, requestBuilder=requestBuilder,
     225            credentials=credentials)
     226      except HttpError as e:
     227        if e.resp.status == http_client.NOT_FOUND:
     228          continue
     229        else:
     230          raise e
     231
     232    raise UnknownApiNameOrVersion(
     233          "name: %s  version: %s" % (serviceName, version))
     234
     235
     236 -def _retrieve_discovery_doc(url, http, cache_discovery, cache=None):
     237    """Retrieves the discovery_doc from cache or the internet.
     238
     239    Args:
     240      url: string, the URL of the discovery document.
     241      http: httplib2.Http, An instance of httplib2.Http or something that acts
     242        like it through which HTTP requests will be made.
     243      cache_discovery: Boolean, whether or not to cache the discovery doc.
     244      cache: googleapiclient.discovery_cache.base.Cache, an optional cache
     245        object for the discovery documents.
     246
     247    Returns:
     248      A unicode string representation of the discovery document.
     249    """
     250    if cache_discovery:
     251      from . import discovery_cache
     252      from .discovery_cache import base
     253      if cache is None:
     254        cache = discovery_cache.autodetect()
     255      if cache:
     256        content = cache.get(url)
     257        if content:
     258          return content
     259
     260    actual_url = url
     261    # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
     262    # variable that contains the network address of the client sending the
     263    # request. If it exists then add that to the request for the discovery
     264    # document to avoid exceeding the quota on discovery requests.
     265    if 'REMOTE_ADDR' in os.environ:
     266      actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR'])
     267    logger.info('URL being requested: GET %s', actual_url)
     268
     269    resp, content = http.request(actual_url)
     270
     271    if resp.status >= 400:
     272      raise HttpError(resp, content, uri=actual_url)
     273
     274    try:
     275      content = content.decode('utf-8')
     276    except AttributeError:
     277      pass
     278
     279    try:
     280      service = json.loads(content)
     281    except ValueError as e:
     282      logger.error('Failed to parse as JSON: ' + content)
     283      raise InvalidJsonError()
     284    if cache_discovery and cache:
     285      cache.set(url, content)
     286    return content
     287
     288
     289  @positional(1)
     290 -def build_from_document(
     291      service,
     292      base=None,
     293      future=None,
     294      http=None,
     295      developerKey=None,
     296      model=None,
     297      requestBuilder=HttpRequest,
     298      credentials=None):
     299    """Create a Resource for interacting with an API.
     300
     301    Same as `build()`, but constructs the Resource object from a discovery
     302    document that is it given, as opposed to retrieving one over HTTP.
     303
     304    Args:
     305      service: string or object, the JSON discovery document describing the API.
     306        The value passed in may either be the JSON string or the deserialized
     307        JSON.
     308      base: string, base URI for all HTTP requests, usually the discovery URI.
     309        This parameter is no longer used as rootUrl and servicePath are included
     310        within the discovery document. (deprecated)
     311      future: string, discovery document with future capabilities (deprecated).
     312      http: httplib2.Http, An instance of httplib2.Http or something that acts
     313        like it that HTTP requests will be made through.
     314      developerKey: string, Key for controlling API usage, generated
     315        from the API Console.
     316      model: Model class instance that serializes and de-serializes requests and
     317        responses.
     318      requestBuilder: Takes an http request and packages it up to be executed.
     319      credentials: object, credentials to be used for authentication.
     320
     321    Returns:
     322      A Resource object with methods for interacting with the service.
     323    """
     324
     325    if http is None:
     326      http = httplib2.Http()
     327
     328    # future is no longer used.
     329    future = {}
     330
     331    if isinstance(service, six.string_types):
     332      service = json.loads(service)
     333
     334    if  'rootUrl' not in service and (isinstance(http, (HttpMock,
     335                                                        HttpMockSequence))):
     336        logger.error("You are using HttpMock or HttpMockSequence without" +
     337                     "having the service discovery doc in cache. Try calling " +
     338                     "build() without mocking once first to populate the " +
     339                     "cache.")
     340        raise InvalidJsonError()
     341
     342    base = urljoin(service['rootUrl'], service['servicePath'])
     343    schema = Schemas(service)
     344
     345    if credentials:
     346      # If credentials were passed in, we could have two cases:
     347      # 1. the scopes were specified, in which case the given credentials
     348      #    are used for authorizing the http;
     349      # 2. the scopes were not provided (meaning the Application Default
     350      #    Credentials are to be used). In this case, the Application Default
     351      #    Credentials are built and used instead of the original credentials.
     352      #    If there are no scopes found (meaning the given service requires no
     353      #    authentication), there is no authorization of the http.
     354      if (isinstance(credentials, GoogleCredentials) and
     355          credentials.create_scoped_required()):
     356        scopes = service.get('auth', {}).get('oauth2', {}).get('scopes', {})
     357        if scopes:
     358          credentials = credentials.create_scoped(list(scopes.keys()))
     359        else:
     360          # No need to authorize the http object
     361          # if the service does not require authentication.
     362          credentials = None
     363
     364      if credentials:
     365        http = credentials.authorize(http)
     366
     367    if model is None:
     368      features = service.get('features', [])
     369      model = JsonModel('dataWrapper' in features)
     370    return Resource(http=http, baseUrl=base, model=model,
     371                    developerKey=developerKey, requestBuilder=requestBuilder,
     372                    resourceDesc=service, rootDesc=service, schema=schema)
     373
     374
     375 -def _cast(value, schema_type):
     376    """Convert value to a string based on JSON Schema type.
     377
     378    See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
     379    JSON Schema.
     380
     381    Args:
     382      value: any, the value to convert
     383      schema_type: string, the type that value should be interpreted as
     384
     385    Returns:
     386      A string representation of 'value' based on the schema_type.
     387    """
     388    if schema_type == 'string':
     389      if type(value) == type('') or type(value) == type(u''):
     390        return value
     391      else:
     392        return str(value)
     393    elif schema_type == 'integer':
     394      return str(int(value))
     395    elif schema_type == 'number':
     396      return str(float(value))
     397    elif schema_type == 'boolean':
     398      return str(bool(value)).lower()
     399    else:
     400      if type(value) == type('') or type(value) == type(u''):
     401        return value
     402      else:
     403        return str(value)
     404
     405
     406 -def _media_size_to_long(maxSize):
     407    """Convert a string media size, such as 10GB or 3TB into an integer.
     408
     409    Args:
     410      maxSize: string, size as a string, such as 2MB or 7GB.
     411
     412    Returns:
     413      The size as an integer value.
     414    """
     415    if len(maxSize) < 2:
     416      return 0
     417    units = maxSize[-2:].upper()
     418    bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units)
     419    if bit_shift is not None:
     420      return int(maxSize[:-2]) << bit_shift
     421    else:
     422      return int(maxSize)
     423
     424
     425 -def _media_path_url_from_info(root_desc, path_url):
     426    """Creates an absolute media path URL.
     427
     428    Constructed using the API root URI and service path from the discovery
     429    document and the relative path for the API method.
     430
     431    Args:
     432      root_desc: Dictionary; the entire original deserialized discovery document.
     433      path_url: String; the relative URL for the API method. Relative to the API
     434          root, which is specified in the discovery document.
     435
     436    Returns:
     437      String; the absolute URI for media upload for the API method.
     438    """
     439    return '%(root)supload/%(service_path)s%(path)s' % {
     440        'root': root_desc['rootUrl'],
     441        'service_path': root_desc['servicePath'],
     442        'path': path_url,
     443    }
     444
     445
     446 -def _fix_up_parameters(method_desc, root_desc, http_method):
     447    """Updates parameters of an API method with values specific to this library.
     448
     449    Specifically, adds whatever global parameters are specified by the API to the
     450    parameters for the individual method. Also adds parameters which don't
     451    appear in the discovery document, but are available to all discovery based
     452    APIs (these are listed in STACK_QUERY_PARAMETERS).
     453
     454    SIDE EFFECTS: This updates the parameters dictionary object in the method
     455    description.
     456
     457    Args:
     458      method_desc: Dictionary with metadata describing an API method. Value comes
     459          from the dictionary of methods stored in the 'methods' key in the
     460          deserialized discovery document.
     461      root_desc: Dictionary; the entire original deserialized discovery document.
     462      http_method: String; the HTTP method used to call the API method described
     463          in method_desc.
     464
     465    Returns:
     466      The updated Dictionary stored in the 'parameters' key of the method
     467          description dictionary.
     468    """
     469    parameters = method_desc.setdefault('parameters', {})
     470
     471    # Add in the parameters common to all methods.
     472    for name, description in six.iteritems(root_desc.get('parameters', {})):
     473      parameters[name] = description
     474
     475    # Add in undocumented query parameters.
     476    for name in STACK_QUERY_PARAMETERS:
     477      parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy()
     478
     479    # Add 'body' (our own reserved word) to parameters if the method supports
     480    # a request payload.
     481    if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc:
     482      body = BODY_PARAMETER_DEFAULT_VALUE.copy()
     483      body.update(method_desc['request'])
     484      parameters['body'] = body
     485
     486    return parameters
     487
     488
     489 -def _fix_up_media_upload(method_desc, root_desc, path_url, parameters):
     490    """Adds 'media_body' and 'media_mime_type' parameters if supported by method.
     491
     492    SIDE EFFECTS: If the method supports media upload and has a required body,
     493    sets body to be optional (required=False) instead. Also, if there is a
     494    'mediaUpload' in the method description, adds 'media_upload' key to
     495    parameters.
     496
     497    Args:
     498      method_desc: Dictionary with metadata describing an API method. Value comes
     499          from the dictionary of methods stored in the 'methods' key in the
     500          deserialized discovery document.
     501      root_desc: Dictionary; the entire original deserialized discovery document.
     502      path_url: String; the relative URL for the API method. Relative to the API
     503          root, which is specified in the discovery document.
     504      parameters: A dictionary describing method parameters for method described
     505          in method_desc.
     506
     507    Returns:
     508      Triple (accept, max_size, media_path_url) where:
     509        - accept is a list of strings representing what content types are
     510          accepted for media upload. Defaults to empty list if not in the
     511          discovery document.
     512        - max_size is a long representing the max size in bytes allowed for a
     513          media upload. Defaults to 0L if not in the discovery document.
     514        - media_path_url is a String; the absolute URI for media upload for the
     515          API method. Constructed using the API root URI and service path from
     516          the discovery document and the relative path for the API method. If
     517          media upload is not supported, this is None.
     518    """
     519    media_upload = method_desc.get('mediaUpload', {})
     520    accept = media_upload.get('accept', [])
     521    max_size = _media_size_to_long(media_upload.get('maxSize', ''))
     522    media_path_url = None
     523
     524    if media_upload:
     525      media_path_url = _media_path_url_from_info(root_desc, path_url)
     526      parameters['media_body'] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy()
     527      parameters['media_mime_type'] = MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE.copy()
     528      if 'body' in parameters:
     529        parameters['body']['required'] = False
     530
     531    return accept, max_size, media_path_url
     532
     533
     534 -def _fix_up_method_description(method_desc, root_desc):
     535    """Updates a method description in a discovery document.
     536
     537    SIDE EFFECTS: Changes the parameters dictionary in the method description with
     538    extra parameters which are used locally.
     539
     540    Args:
     541      method_desc: Dictionary with metadata describing an API method. Value comes
     542          from the dictionary of methods stored in the 'methods' key in the
     543          deserialized discovery document.
     544      root_desc: Dictionary; the entire original deserialized discovery document.
     545
     546    Returns:
     547      Tuple (path_url, http_method, method_id, accept, max_size, media_path_url)
     548      where:
     549        - path_url is a String; the relative URL for the API method. Relative to
     550          the API root, which is specified in the discovery document.
     551        - http_method is a String; the HTTP method used to call the API method
     552          described in the method description.
     553        - method_id is a String; the name of the RPC method associated with the
     554          API method, and is in the method description in the 'id' key.
     555        - accept is a list of strings representing what content types are
     556          accepted for media upload. Defaults to empty list if not in the
     557          discovery document.
     558        - max_size is a long representing the max size in bytes allowed for a
     559          media upload. Defaults to 0L if not in the discovery document.
     560        - media_path_url is a String; the absolute URI for media upload for the
     561          API method. Constructed using the API root URI and service path from
     562          the discovery document and the relative path for the API method. If
     563          media upload is not supported, this is None.
     564    """
     565    path_url = method_desc['path']
     566    http_method = method_desc['httpMethod']
     567    method_id = method_desc['id']
     568
     569    parameters = _fix_up_parameters(method_desc, root_desc, http_method)
     570    # Order is important. `_fix_up_media_upload` needs `method_desc` to have a
     571    # 'parameters' key and needs to know if there is a 'body' parameter because it
     572    # also sets a 'media_body' parameter.
     573    accept, max_size, media_path_url = _fix_up_media_upload(
     574        method_desc, root_desc, path_url, parameters)
     575
     576    return path_url, http_method, method_id, accept, max_size, media_path_url
     577
     578
     579 -def _urljoin(base, url):
     580    """Custom urljoin replacement supporting : before / in url."""
     581    # In general, it's unsafe to simply join base and url. However, for
     582    # the case of discovery documents, we know:
     583    #  * base will never contain params, query, or fragment
     584    #  * url will never contain a scheme or net_loc.
     585    # In general, this means we can safely join on /; we just need to
     586    # ensure we end up with precisely one / joining base and url. The
     587    # exception here is the case of media uploads, where url will be an
     588    # absolute url.
     589    if url.startswith('http://') or url.startswith('https://'):
     590      return urljoin(base, url)
     591    new_base = base if base.endswith('/') else base + '/'
     592    new_url = url[1:] if url.startswith('/') else url
     593    return new_base + new_url
     594
     595
     596  # TODO(dhermes): Convert this class to ResourceMethod and make it callable
     597 -class ResourceMethodParameters(object):
     598    """Represents the parameters associated with a method.
     599
     600    Attributes:
     601      argmap: Map from method parameter name (string) to query parameter name
     602          (string).
     603      required_params: List of required parameters (represented by parameter
     604          name as string).
     605      repeated_params: List of repeated parameters (represented by parameter
     606          name as string).
     607      pattern_params: Map from method parameter name (string) to regular
     608          expression (as a string). If the pattern is set for a parameter, the
     609          value for that parameter must match the regular expression.
     610      query_params: List of parameters (represented by parameter name as string)
     611          that will be used in the query string.
     612      path_params: Set of parameters (represented by parameter name as string)
     613          that will be used in the base URL path.
     614      param_types: Map from method parameter name (string) to parameter type. Type
     615          can be any valid JSON schema type; valid values are 'any', 'array',
     616          'boolean', 'integer', 'number', 'object', or 'string'. Reference:
     617          http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1
     618      enum_params: Map from method parameter name (string) to list of strings,
     619         where each list of strings is the list of acceptable enum values.
     620    """
     621
     622 -  def __init__(self, method_desc):
     623      """Constructor for ResourceMethodParameters.
     624
     625      Sets default values and defers to set_parameters to populate.
     626
     627      Args:
     628        method_desc: Dictionary with metadata describing an API method. Value
     629            comes from the dictionary of methods stored in the 'methods' key in
     630            the deserialized discovery document.
     631      """
     632      self.argmap = {}
     633      self.required_params = []
     634      self.repeated_params = []
     635      self.pattern_params = {}
     636      self.query_params = []
     637      # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE
     638      #                parsing is gotten rid of.
     639      self.path_params = set()
     640      self.param_types = {}
     641      self.enum_params = {}
     642
     643      self.set_parameters(method_desc)
     644
     645 -  def set_parameters(self, method_desc):
     646      """Populates maps and lists based on method description.
     647
     648      Iterates through each parameter for the method and parses the values from
     649      the parameter dictionary.
     650
     651      Args:
     652        method_desc: Dictionary with metadata describing an API method. Value
     653            comes from the dictionary of methods stored in the 'methods' key in
     654            the deserialized discovery document.
     655      """
     656      for arg, desc in six.iteritems(method_desc.get('parameters', {})):
     657        param = key2param(arg)
     658        self.argmap[param] = arg
     659
     660        if desc.get('pattern'):
     661          self.pattern_params[param] = desc['pattern']
     662        if desc.get('enum'):
     663          self.enum_params[param] = desc['enum']
     664        if desc.get('required'):
     665          self.required_params.append(param)
     666        if desc.get('repeated'):
     667          self.repeated_params.append(param)
     668        if desc.get('location') == 'query':
     669          self.query_params.append(param)
     670        if desc.get('location') == 'path':
     671          self.path_params.add(param)
     672        self.param_types[param] = desc.get('type', 'string')
     673
     674      # TODO(dhermes): Determine if this is still necessary. Discovery based APIs
     675      #                should have all path parameters already marked with
     676      #                'location: path'.
     677      for match in URITEMPLATE.finditer(method_desc['path']):
     678        for namematch in VARNAME.finditer(match.group(0)):
     679          name = key2param(namematch.group(0))
     680          self.path_params.add(name)
     681          if name in self.query_params:
     682            self.query_params.remove(name)
     683
     684
     685 -def createMethod(methodName, methodDesc, rootDesc, schema):
     686    """Creates a method for attaching to a Resource.
     687
     688    Args:
     689      methodName: string, name of the method to use.
     690      methodDesc: object, fragment of deserialized discovery document that
     691        describes the method.
     692      rootDesc: object, the entire deserialized discovery document.
     693      schema: object, mapping of schema names to schema descriptions.
     694    """
     695    methodName = fix_method_name(methodName)
     696    (pathUrl, httpMethod, methodId, accept,
     697     maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc)
     698
     699    parameters = ResourceMethodParameters(methodDesc)
     700
     701    def method(self, **kwargs):
     702      # Don't bother with doc string, it will be over-written by createMethod.
     703
     704      for name in six.iterkeys(kwargs):
     705        if name not in parameters.argmap:
     706          raise TypeError('Got an unexpected keyword argument "%s"' % name)
     707
     708      # Remove args that have a value of None.
     709      keys = list(kwargs.keys())
     710      for name in keys:
     711        if kwargs[name] is None:
     712          del kwargs[name]
     713
     714      for name in parameters.required_params:
     715        if name not in kwargs:
     716          raise TypeError('Missing required parameter "%s"' % name)
     717
     718      for name, regex in six.iteritems(parameters.pattern_params):
     719        if name in kwargs:
     720          if isinstance(kwargs[name], six.string_types):
     721            pvalues = [kwargs[name]]
     722          else:
     723            pvalues = kwargs[name]
     724          for pvalue in pvalues:
     725            if re.match(regex, pvalue) is None:
     726              raise TypeError(
     727                  'Parameter "%s" value "%s" does not match the pattern "%s"' %
     728                  (name, pvalue, regex))
     729
     730      for name, enums in six.iteritems(parameters.enum_params):
     731        if name in kwargs:
     732          # We need to handle the case of a repeated enum
     733          # name differently, since we want to handle both
     734          # arg='value' and arg=['value1', 'value2']
     735          if (name in parameters.repeated_params and
     736              not isinstance(kwargs[name], six.string_types)):
     737            values = kwargs[name]
     738          else:
     739            values = [kwargs[name]]
     740          for value in values:
     741            if value not in enums:
     742              raise TypeError(
     743                  'Parameter "%s" value "%s" is not an allowed value in "%s"' %
     744                  (name, value, str(enums)))
     745
     746      actual_query_params = {}
     747      actual_path_params = {}
     748      for key, value in six.iteritems(kwargs):
     749        to_type = parameters.param_types.get(key, 'string')
     750        # For repeated parameters we cast each member of the list.
     751        if key in parameters.repeated_params and type(value) == type([]):
     752          cast_value = [_cast(x, to_type) for x in value]
     753        else:
     754          cast_value = _cast(value, to_type)
     755        if key in parameters.query_params:
     756          actual_query_params[parameters.argmap[key]] = cast_value
     757        if key in parameters.path_params:
     758          actual_path_params[parameters.argmap[key]] = cast_value
     759      body_value = kwargs.get('body', None)
     760      media_filename = kwargs.get('media_body', None)
     761      media_mime_type = kwargs.get('media_mime_type', None)
     762
     763      if self._developerKey:
     764        actual_query_params['key'] = self._developerKey
     765
     766      model = self._model
     767      if methodName.endswith('_media'):
     768        model = MediaModel()
     769      elif 'response' not in methodDesc:
     770        model = RawModel()
     771
     772      headers = {}
     773      headers, params, query, body = model.request(headers,
     774          actual_path_params, actual_query_params, body_value)
     775
     776      expanded_url = uritemplate.expand(pathUrl, params)
     777      url = _urljoin(self._baseUrl, expanded_url + query)
     778
     779      resumable = None
     780      multipart_boundary = ''
     781
     782      if media_filename:
     783        # Ensure we end up with a valid MediaUpload object.
     784        if isinstance(media_filename, six.string_types):
     785          if media_mime_type is None:
     786            logger.warning(
     787                'media_mime_type argument not specified: trying to auto-detect for %s',
     788                media_filename)
     789            media_mime_type, _ = mimetypes.guess_type(media_filename)
     790          if media_mime_type is None:
     791            raise UnknownFileType(media_filename)
     792          if not mimeparse.best_match([media_mime_type], ','.join(accept)):
     793            raise UnacceptableMimeTypeError(media_mime_type)
     794          media_upload = MediaFileUpload(media_filename,
     795                                         mimetype=media_mime_type)
     796        elif isinstance(media_filename, MediaUpload):
     797          media_upload = media_filename
     798        else:
     799          raise TypeError('media_filename must be str or MediaUpload.')
     800
     801        # Check the maxSize
     802        if media_upload.size() is not None and media_upload.size() > maxSize > 0:
     803          raise MediaUploadSizeError("Media larger than: %s" % maxSize)
     804
     805        # Use the media path uri for media uploads
     806        expanded_url = uritemplate.expand(mediaPathUrl, params)
     807        url = _urljoin(self._baseUrl, expanded_url + query)
     808        if media_upload.resumable():
     809          url = _add_query_parameter(url, 'uploadType', 'resumable')
     810
     811        if media_upload.resumable():
     812          # This is all we need to do for resumable, if the body exists it gets
     813          # sent in the first request, otherwise an empty body is sent.
     814          resumable = media_upload
     815        else:
     816          # A non-resumable upload
     817          if body is None:
     818            # This is a simple media upload
     819            headers['content-type'] = media_upload.mimetype()
     820            body = media_upload.getbytes(0, media_upload.size())
     821            url = _add_query_parameter(url, 'uploadType', 'media')
     822          else:
     823            # This is a multipart/related upload.
     824            msgRoot = MIMEMultipart('related')
     825            # msgRoot should not write out it's own headers
     826            setattr(msgRoot, '_write_headers', lambda self: None)
     827
     828            # attach the body as one part
     829            msg = MIMENonMultipart(*headers['content-type'].split('/'))
     830            msg.set_payload(body)
     831            msgRoot.attach(msg)
     832
     833            # attach the media as the second part
     834            msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
     835            msg['Content-Transfer-Encoding'] = 'binary'
     836
     837            payload = media_upload.getbytes(0, media_upload.size())
     838            msg.set_payload(payload)
     839            msgRoot.attach(msg)
     840            # encode the body: note that we can't use `as_string`, because
     841            # it plays games with `From ` lines.
     842            fp = BytesIO()
     843            g = _BytesGenerator(fp, mangle_from_=False)
     844            g.flatten(msgRoot, unixfrom=False)
     845            body = fp.getvalue()
     846
     847            multipart_boundary = msgRoot.get_boundary()
     848            headers['content-type'] = ('multipart/related; '
     849                                       'boundary="%s"') % multipart_boundary
     850            url = _add_query_parameter(url, 'uploadType', 'multipart')
     851
     852      logger.info('URL being requested: %s %s' % (httpMethod,url))
     853      return self._requestBuilder(self._http,
     854                                  model.response,
     855                                  url,
     856                                  method=httpMethod,
     857                                  body=body,
     858                                  headers=headers,
     859                                  methodId=methodId,
     860                                  resumable=resumable)
     861
     862    docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
     863    if len(parameters.argmap) > 0:
     864      docs.append('Args:\n')
     865
     866    # Skip undocumented params and params common to all methods.
     867    skip_parameters = list(rootDesc.get('parameters', {}).keys())
     868    skip_parameters.extend(STACK_QUERY_PARAMETERS)
     869
     870    all_args = list(parameters.argmap.keys())
     871    args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
     872
     873    # Move body to the front of the line.
     874    if 'body' in all_args:
     875      args_ordered.append('body')
     876
     877    for name in all_args:
     878      if name not in args_ordered:
     879        args_ordered.append(name)
     880
     881    for arg in args_ordered:
     882      if arg in skip_parameters:
     883        continue
     884
     885      repeated = ''
     886      if arg in parameters.repeated_params:
     887        repeated = ' (repeated)'
     888      required = ''
     889      if arg in parameters.required_params:
     890        required = ' (required)'
     891      paramdesc = methodDesc['parameters'][parameters.argmap[arg]]
     892      paramdoc = paramdesc.get('description', 'A parameter')
     893      if '$ref' in paramdesc:
     894        docs.append(
     895            ('  %s: object, %s%s%s\n    The object takes the'
     896            ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
     897              schema.prettyPrintByName(paramdesc['$ref'])))
     898      else:
     899        paramtype = paramdesc.get('type', 'string')
     900        docs.append('  %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
     901                                            repeated))
     902      enum = paramdesc.get('enum', [])
     903      enumDesc = paramdesc.get('enumDescriptions', [])
     904      if enum and enumDesc:
     905        docs.append('    Allowed values\n')
     906        for (name, desc) in zip(enum, enumDesc):
     907          docs.append('      %s - %s\n' % (name, desc))
     908    if 'response' in methodDesc:
     909      if methodName.endswith('_media'):
     910        docs.append('\nReturns:\n  The media object as a string.\n\n    ')
     911      else:
     912        docs.append('\nReturns:\n  An object of the form:\n\n    ')
     913        docs.append(schema.prettyPrintSchema(methodDesc['response']))
     914
     915    setattr(method, '__doc__', ''.join(docs))
     916    return (methodName, method)
     917
     918
     919 -def createNextMethod(methodName):
     920    """Creates any _next methods for attaching to a Resource.
     921
     922    The _next methods allow for easy iteration through list() responses.
     923
     924    Args:
     925      methodName: string, name of the method to use.
     926    """
     927    methodName = fix_method_name(methodName)
     928
     929    def methodNext(self, previous_request, previous_response):
     930      """Retrieves the next page of results.
     931
     932  Args:
     933    previous_request: The request for the previous page. (required)
     934    previous_response: The response from the request for the previous page. (required)
     935
     936  Returns:
     937    A request object that you can call 'execute()' on to request the next
     938    page. Returns None if there are no more items in the collection.
     939      """
     940      # Retrieve nextPageToken from previous_response
     941      # Use as pageToken in previous_request to create new request.
     942
     943      if 'nextPageToken' not in previous_response or not previous_response['nextPageToken']:
     944        return None
     945
     946      request = copy.copy(previous_request)
     947
     948      pageToken = previous_response['nextPageToken']
     949      parsed = list(urlparse(request.uri))
     950      q = parse_qsl(parsed[4])
     951
     952      # Find and remove old 'pageToken' value from URI
     953      newq = [(key, value) for (key, value) in q if key != 'pageToken']
     954      newq.append(('pageToken', pageToken))
     955      parsed[4] = urlencode(newq)
     956      uri = urlunparse(parsed)
     957
     958      request.uri = uri
     959
     960      logger.info('URL being requested: %s %s' % (methodName,uri))
     961
     962      return request
     963
     964    return (methodName, methodNext)
     965
     966
     967 -class Resource(object):
     968    """A class for interacting with a resource."""
     969
     970 -  def __init__(self, http, baseUrl, model, requestBuilder, developerKey,
     971                 resourceDesc, rootDesc, schema):
     972      """Build a Resource from the API description.
     973
     974      Args:
     975        http: httplib2.Http, Object to make http requests with.
     976        baseUrl: string, base URL for the API. All requests are relative to this
     977            URI.
     978        model: googleapiclient.Model, converts to and from the wire format.
     979        requestBuilder: class or callable that instantiates an
     980            googleapiclient.HttpRequest object.
     981        developerKey: string, key obtained from
     982            https://code.google.com/apis/console
     983        resourceDesc: object, section of deserialized discovery document that
     984            describes a resource. Note that the top level discovery document
     985            is considered a resource.
     986        rootDesc: object, the entire deserialized discovery document.
     987        schema: object, mapping of schema names to schema descriptions.
     988      """
     989      self._dynamic_attrs = []
     990
     991      self._http = http
     992      self._baseUrl = baseUrl
     993      self._model = model
     994      self._developerKey = developerKey
     995      self._requestBuilder = requestBuilder
     996      self._resourceDesc = resourceDesc
     997      self._rootDesc = rootDesc
     998      self._schema = schema
     999
    1000      self._set_service_methods()
    1001
    1002 -  def _set_dynamic_attr(self, attr_name, value):
    1003      """Sets an instance attribute and tracks it in a list of dynamic attributes.
    1004
    1005      Args:
    1006        attr_name: string; The name of the attribute to be set
    1007        value: The value being set on the object and tracked in the dynamic cache.
    1008      """
    1009      self._dynamic_attrs.append(attr_name)
    1010      self.__dict__[attr_name] = value
    1011
    1012 -  def __getstate__(self):
    1013      """Trim the state down to something that can be pickled.
    1014
    1015      Uses the fact that the instance variable _dynamic_attrs holds attrs that
    1016      will be wiped and restored on pickle serialization.
    1017      """
    1018      state_dict = copy.copy(self.__dict__)
    1019      for dynamic_attr in self._dynamic_attrs:
    1020        del state_dict[dynamic_attr]
    1021      del state_dict['_dynamic_attrs']
    1022      return state_dict
    1023
    1024 -  def __setstate__(self, state):
    1025      """Reconstitute the state of the object from being pickled.
    1026
    1027      Uses the fact that the instance variable _dynamic_attrs holds attrs that
    1028      will be wiped and restored on pickle serialization.
    1029      """
    1030      self.__dict__.update(state)
    1031      self._dynamic_attrs = []
    1032      self._set_service_methods()
    1033
    1034 -  def _set_service_methods(self):
    1035      self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema)
    1036      self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema)
    1037      self._add_next_methods(self._resourceDesc, self._schema)
    1038
    1039 -  def _add_basic_methods(self, resourceDesc, rootDesc, schema):
    1040      # If this is the root Resource, add a new_batch_http_request() method.
    1041      if resourceDesc == rootDesc:
    1042        batch_uri = '%s%s' % (
    1043          rootDesc['rootUrl'], rootDesc.get('batchPath', 'batch'))
    1044        def new_batch_http_request(callback=None):
    1045          """Create a BatchHttpRequest object based on the discovery document.
    1046
    1047          Args:
    1048            callback: callable, A callback to be called for each response, of the
    1049              form callback(id, response, exception). The first parameter is the
    1050              request id, and the second is the deserialized response object. The
    1051              third is an apiclient.errors.HttpError exception object if an HTTP
    1052              error occurred while processing the request, or None if no error
    1053              occurred.
    1054
    1055          Returns:
    1056            A BatchHttpRequest object based on the discovery document.
    1057          """
    1058          return BatchHttpRequest(callback=callback, batch_uri=batch_uri)
    1059        self._set_dynamic_attr('new_batch_http_request', new_batch_http_request)
    1060
    1061      # Add basic methods to Resource
    1062      if 'methods' in resourceDesc:
    1063        for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1064          fixedMethodName, method = createMethod(
    1065              methodName, methodDesc, rootDesc, schema)
    1066          self._set_dynamic_attr(fixedMethodName,
    1067                                 method.__get__(self, self.__class__))
    1068          # Add in _media methods. The functionality of the attached method will
    1069          # change when it sees that the method name ends in _media.
    1070          if methodDesc.get('supportsMediaDownload', False):
    1071            fixedMethodName, method = createMethod(
    1072                methodName + '_media', methodDesc, rootDesc, schema)
    1073            self._set_dynamic_attr(fixedMethodName,
    1074                                   method.__get__(self, self.__class__))
    1075
    1076 -  def _add_nested_resources(self, resourceDesc, rootDesc, schema):
    1077      # Add in nested resources
    1078      if 'resources' in resourceDesc:
    1079
    1080        def createResourceMethod(methodName, methodDesc):
    1081          """Create a method on the Resource to access a nested Resource.
    1082
    1083          Args:
    1084            methodName: string, name of the method to use.
    1085            methodDesc: object, fragment of deserialized discovery document that
    1086              describes the method.
    1087          """
    1088          methodName = fix_method_name(methodName)
    1089
    1090          def methodResource(self):
    1091            return Resource(http=self._http, baseUrl=self._baseUrl,
    1092                            model=self._model, developerKey=self._developerKey,
    1093                            requestBuilder=self._requestBuilder,
    1094                            resourceDesc=methodDesc, rootDesc=rootDesc,
    1095                            schema=schema)
    1096
    1097          setattr(methodResource, '__doc__', 'A collection resource.')
    1098          setattr(methodResource, '__is_resource__', True)
    1099
    1100          return (methodName, methodResource)
    1101
    1102        for methodName, methodDesc in six.iteritems(resourceDesc['resources']):
    1103          fixedMethodName, method = createResourceMethod(methodName, methodDesc)
    1104          self._set_dynamic_attr(fixedMethodName,
    1105                                 method.__get__(self, self.__class__))
    1106
    1107 -  def _add_next_methods(self, resourceDesc, schema):
    1108      # Add _next() methods
    1109      # Look for response bodies in schema that contain nextPageToken, and methods
    1110      # that take a pageToken parameter.
    1111      if 'methods' in resourceDesc:
    1112        for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1113          if 'response' in methodDesc:
    1114            responseSchema = methodDesc['response']
    1115            if '$ref' in responseSchema:
    1116              responseSchema = schema.get(responseSchema['$ref'])
    1117            hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
    1118                                                                     {})
    1119            hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
    1120            if hasNextPageToken and hasPageToken:
    1121              fixedMethodName, method = createNextMethod(methodName + '_next')
    1122              self._set_dynamic_attr(fixedMethodName,
    1123                                     method.__get__(self, self.__class__))
    1124