googleapiclient.http

Package googleapiclient :: Module http |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.http

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Classes to encapsulate a single HTTP request.
      16
      17  The classes implement a command pattern, with every
      18  object supporting an execute() method that does the
      19  actuall HTTP request.
      20  """
      21  from __future__ import absolute_import
      22  import six
      23  from six.moves import range
      24
      25  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      26
      27  from six import BytesIO, StringIO
      28  from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote
      29
      30  import base64
      31  import copy
      32  import gzip
      33  import httplib2
      34  import json
      35  import logging
      36  import mimetypes
      37  import os
      38  import random
      39  import sys
      40  import time
      41  import uuid
      42
      43  from email.generator import Generator
      44  from email.mime.multipart import MIMEMultipart
      45  from email.mime.nonmultipart import MIMENonMultipart
      46  from email.parser import FeedParser
      47
      48  from googleapiclient import mimeparse
      49  from googleapiclient.errors import BatchError
      50  from googleapiclient.errors import HttpError
      51  from googleapiclient.errors import InvalidChunkSizeError
      52  from googleapiclient.errors import ResumableUploadError
      53  from googleapiclient.errors import UnexpectedBodyError
      54  from googleapiclient.errors import UnexpectedMethodError
      55  from googleapiclient.model import JsonModel
      56  from oauth2client import util
      57
      58
      59  DEFAULT_CHUNK_SIZE = 512*1024
      60
      61  MAX_URI_LENGTH = 2048
      62
      63
      64 -class MediaUploadProgress(object):
      65    """Status of a resumable upload."""
      66
      67 -  def __init__(self, resumable_progress, total_size):
      68      """Constructor.
      69
      70      Args:
      71        resumable_progress: int, bytes sent so far.
      72        total_size: int, total bytes in complete upload, or None if the total
      73          upload size isn't known ahead of time.
      74      """
      75      self.resumable_progress = resumable_progress
      76      self.total_size = total_size
      77
      78 -  def progress(self):
      79      """Percent of upload completed, as a float.
      80
      81      Returns:
      82        the percentage complete as a float, returning 0.0 if the total size of
      83        the upload is unknown.
      84      """
      85      if self.total_size is not None:
      86        return float(self.resumable_progress) / float(self.total_size)
      87      else:
      88        return 0.0
      89
      90
      91 -class MediaDownloadProgress(object):
      92    """Status of a resumable download."""
      93
      94 -  def __init__(self, resumable_progress, total_size):
      95      """Constructor.
      96
      97      Args:
      98        resumable_progress: int, bytes received so far.
      99        total_size: int, total bytes in complete download.
     100      """
     101      self.resumable_progress = resumable_progress
     102      self.total_size = total_size
     103
     104 -  def progress(self):
     105      """Percent of download completed, as a float.
     106
     107      Returns:
     108        the percentage complete as a float, returning 0.0 if the total size of
     109        the download is unknown.
     110      """
     111      if self.total_size is not None:
     112        return float(self.resumable_progress) / float(self.total_size)
     113      else:
     114        return 0.0
     115
     116
     117 -class MediaUpload(object):
     118    """Describes a media object to upload.
     119
     120    Base class that defines the interface of MediaUpload subclasses.
     121
     122    Note that subclasses of MediaUpload may allow you to control the chunksize
     123    when uploading a media object. It is important to keep the size of the chunk
     124    as large as possible to keep the upload efficient. Other factors may influence
     125    the size of the chunk you use, particularly if you are working in an
     126    environment where individual HTTP requests may have a hardcoded time limit,
     127    such as under certain classes of requests under Google App Engine.
     128
     129    Streams are io.Base compatible objects that support seek(). Some MediaUpload
     130    subclasses support using streams directly to upload data. Support for
     131    streaming may be indicated by a MediaUpload sub-class and if appropriate for a
     132    platform that stream will be used for uploading the media object. The support
     133    for streaming is indicated by has_stream() returning True. The stream() method
     134    should return an io.Base object that supports seek(). On platforms where the
     135    underlying httplib module supports streaming, for example Python 2.6 and
     136    later, the stream will be passed into the http library which will result in
     137    less memory being used and possibly faster uploads.
     138
     139    If you need to upload media that can't be uploaded using any of the existing
     140    MediaUpload sub-class then you can sub-class MediaUpload for your particular
     141    needs.
     142    """
     143
     144 -  def chunksize(self):
     145      """Chunk size for resumable uploads.
     146
     147      Returns:
     148        Chunk size in bytes.
     149      """
     150      raise NotImplementedError()
     151
     152 -  def mimetype(self):
     153      """Mime type of the body.
     154
     155      Returns:
     156        Mime type.
     157      """
     158      return 'application/octet-stream'
     159
     160 -  def size(self):
     161      """Size of upload.
     162
     163      Returns:
     164        Size of the body, or None of the size is unknown.
     165      """
     166      return None
     167
     168 -  def resumable(self):
     169      """Whether this upload is resumable.
     170
     171      Returns:
     172        True if resumable upload or False.
     173      """
     174      return False
     175
     176 -  def getbytes(self, begin, end):
     177      """Get bytes from the media.
     178
     179      Args:
     180        begin: int, offset from beginning of file.
     181        length: int, number of bytes to read, starting at begin.
     182
     183      Returns:
     184        A string of bytes read. May be shorter than length if EOF was reached
     185        first.
     186      """
     187      raise NotImplementedError()
     188
     189 -  def has_stream(self):
     190      """Does the underlying upload support a streaming interface.
     191
     192      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     193      seekable() returns True.
     194
     195      Returns:
     196        True if the call to stream() will return an instance of a seekable io.Base
     197        subclass.
     198      """
     199      return False
     200
     201 -  def stream(self):
     202      """A stream interface to the data being uploaded.
     203
     204      Returns:
     205        The returned value is an io.IOBase subclass that supports seek, i.e.
     206        seekable() returns True.
     207      """
     208      raise NotImplementedError()
     209
     210    @util.positional(1)
     211 -  def _to_json(self, strip=None):
     212      """Utility function for creating a JSON representation of a MediaUpload.
     213
     214      Args:
     215        strip: array, An array of names of members to not include in the JSON.
     216
     217      Returns:
     218         string, a JSON representation of this instance, suitable to pass to
     219         from_json().
     220      """
     221      t = type(self)
     222      d = copy.copy(self.__dict__)
     223      if strip is not None:
     224        for member in strip:
     225          del d[member]
     226      d['_class'] = t.__name__
     227      d['_module'] = t.__module__
     228      return json.dumps(d)
     229
     230 -  def to_json(self):
     231      """Create a JSON representation of an instance of MediaUpload.
     232
     233      Returns:
     234         string, a JSON representation of this instance, suitable to pass to
     235         from_json().
     236      """
     237      return self._to_json()
     238
     239    @classmethod
     240 -  def new_from_json(cls, s):
     241      """Utility class method to instantiate a MediaUpload subclass from a JSON
     242      representation produced by to_json().
     243
     244      Args:
     245        s: string, JSON from to_json().
     246
     247      Returns:
     248        An instance of the subclass of MediaUpload that was serialized with
     249        to_json().
     250      """
     251      data = json.loads(s)
     252      # Find and call the right classmethod from_json() to restore the object.
     253      module = data['_module']
     254      m = __import__(module, fromlist=module.split('.')[:-1])
     255      kls = getattr(m, data['_class'])
     256      from_json = getattr(kls, 'from_json')
     257      return from_json(s)
     258
     259
     260 -class MediaIoBaseUpload(MediaUpload):
     261    """A MediaUpload for a io.Base objects.
     262
     263    Note that the Python file object is compatible with io.Base and can be used
     264    with this class also.
     265
     266      fh = BytesIO('...Some data to upload...')
     267      media = MediaIoBaseUpload(fh, mimetype='image/png',
     268        chunksize=1024*1024, resumable=True)
     269      farm.animals().insert(
     270          id='cow',
     271          name='cow.png',
     272          media_body=media).execute()
     273
     274    Depending on the platform you are working on, you may pass -1 as the
     275    chunksize, which indicates that the entire file should be uploaded in a single
     276    request. If the underlying platform supports streams, such as Python 2.6 or
     277    later, then this can be very efficient as it avoids multiple connections, and
     278    also avoids loading the entire file into memory before sending it. Note that
     279    Google App Engine has a 5MB limit on request size, so you should never set
     280    your chunksize larger than 5MB, or to -1.
     281    """
     282
     283    @util.positional(3)
     284 -  def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
     285        resumable=False):
     286      """Constructor.
     287
     288      Args:
     289        fd: io.Base or file object, The source of the bytes to upload. MUST be
     290          opened in blocking mode, do not use streams opened in non-blocking mode.
     291          The given stream must be seekable, that is, it must be able to call
     292          seek() on fd.
     293        mimetype: string, Mime-type of the file.
     294        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     295          used if resumable=True. Pass in a value of -1 if the file is to be
     296          uploaded as a single chunk. Note that Google App Engine has a 5MB limit
     297          on request size, so you should never set your chunksize larger than 5MB,
     298          or to -1.
     299        resumable: bool, True if this is a resumable upload. False means upload
     300          in a single request.
     301      """
     302      super(MediaIoBaseUpload, self).__init__()
     303      self._fd = fd
     304      self._mimetype = mimetype
     305      if not (chunksize == -1 or chunksize > 0):
     306        raise InvalidChunkSizeError()
     307      self._chunksize = chunksize
     308      self._resumable = resumable
     309
     310      self._fd.seek(0, os.SEEK_END)
     311      self._size = self._fd.tell()
     312
     313 -  def chunksize(self):
     314      """Chunk size for resumable uploads.
     315
     316      Returns:
     317        Chunk size in bytes.
     318      """
     319      return self._chunksize
     320
     321 -  def mimetype(self):
     322      """Mime type of the body.
     323
     324      Returns:
     325        Mime type.
     326      """
     327      return self._mimetype
     328
     329 -  def size(self):
     330      """Size of upload.
     331
     332      Returns:
     333        Size of the body, or None of the size is unknown.
     334      """
     335      return self._size
     336
     337 -  def resumable(self):
     338      """Whether this upload is resumable.
     339
     340      Returns:
     341        True if resumable upload or False.
     342      """
     343      return self._resumable
     344
     345 -  def getbytes(self, begin, length):
     346      """Get bytes from the media.
     347
     348      Args:
     349        begin: int, offset from beginning of file.
     350        length: int, number of bytes to read, starting at begin.
     351
     352      Returns:
     353        A string of bytes read. May be shorted than length if EOF was reached
     354        first.
     355      """
     356      self._fd.seek(begin)
     357      return self._fd.read(length)
     358
     359 -  def has_stream(self):
     360      """Does the underlying upload support a streaming interface.
     361
     362      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     363      seekable() returns True.
     364
     365      Returns:
     366        True if the call to stream() will return an instance of a seekable io.Base
     367        subclass.
     368      """
     369      return True
     370
     371 -  def stream(self):
     372      """A stream interface to the data being uploaded.
     373
     374      Returns:
     375        The returned value is an io.IOBase subclass that supports seek, i.e.
     376        seekable() returns True.
     377      """
     378      return self._fd
     379
     380 -  def to_json(self):
     381      """This upload type is not serializable."""
     382      raise NotImplementedError('MediaIoBaseUpload is not serializable.')
     383
     384
     385 -class MediaFileUpload(MediaIoBaseUpload):
     386    """A MediaUpload for a file.
     387
     388    Construct a MediaFileUpload and pass as the media_body parameter of the
     389    method. For example, if we had a service that allowed uploading images:
     390
     391
     392      media = MediaFileUpload('cow.png', mimetype='image/png',
     393        chunksize=1024*1024, resumable=True)
     394      farm.animals().insert(
     395          id='cow',
     396          name='cow.png',
     397          media_body=media).execute()
     398
     399    Depending on the platform you are working on, you may pass -1 as the
     400    chunksize, which indicates that the entire file should be uploaded in a single
     401    request. If the underlying platform supports streams, such as Python 2.6 or
     402    later, then this can be very efficient as it avoids multiple connections, and
     403    also avoids loading the entire file into memory before sending it. Note that
     404    Google App Engine has a 5MB limit on request size, so you should never set
     405    your chunksize larger than 5MB, or to -1.
     406    """
     407
     408    @util.positional(2)
     409 -  def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
     410                 resumable=False):
     411      """Constructor.
     412
     413      Args:
     414        filename: string, Name of the file.
     415        mimetype: string, Mime-type of the file. If None then a mime-type will be
     416          guessed from the file extension.
     417        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     418          used if resumable=True. Pass in a value of -1 if the file is to be
     419          uploaded in a single chunk. Note that Google App Engine has a 5MB limit
     420          on request size, so you should never set your chunksize larger than 5MB,
     421          or to -1.
     422        resumable: bool, True if this is a resumable upload. False means upload
     423          in a single request.
     424      """
     425      self._filename = filename
     426      fd = open(self._filename, 'rb')
     427      if mimetype is None:
     428        (mimetype, encoding) = mimetypes.guess_type(filename)
     429      super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     430                                            resumable=resumable)
     431
     432 -  def to_json(self):
     433      """Creating a JSON representation of an instance of MediaFileUpload.
     434
     435      Returns:
     436         string, a JSON representation of this instance, suitable to pass to
     437         from_json().
     438      """
     439      return self._to_json(strip=['_fd'])
     440
     441    @staticmethod
     442 -  def from_json(s):
     443      d = json.loads(s)
     444      return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
     445                             chunksize=d['_chunksize'], resumable=d['_resumable'])
     446
     447
     448 -class MediaInMemoryUpload(MediaIoBaseUpload):
     449    """MediaUpload for a chunk of bytes.
     450
     451    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     452    the stream.
     453    """
     454
     455    @util.positional(2)
     456 -  def __init__(self, body, mimetype='application/octet-stream',
     457                 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
     458      """Create a new MediaInMemoryUpload.
     459
     460    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     461    the stream.
     462
     463    Args:
     464      body: string, Bytes of body content.
     465      mimetype: string, Mime-type of the file or default of
     466        'application/octet-stream'.
     467      chunksize: int, File will be uploaded in chunks of this many bytes. Only
     468        used if resumable=True.
     469      resumable: bool, True if this is a resumable upload. False means upload
     470        in a single request.
     471      """
     472      fd = BytesIO(body)
     473      super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     474                                                resumable=resumable)
     475
     476
     477 -class MediaIoBaseDownload(object):
     478    """"Download media resources.
     479
     480    Note that the Python file object is compatible with io.Base and can be used
     481    with this class also.
     482
     483
     484    Example:
     485      request = farms.animals().get_media(id='cow')
     486      fh = io.FileIO('cow.png', mode='wb')
     487      downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
     488
     489      done = False
     490      while done is False:
     491        status, done = downloader.next_chunk()
     492        if status:
     493          print "Download %d%%." % int(status.progress() * 100)
     494      print "Download Complete!"
     495    """
     496
     497    @util.positional(3)
     498 -  def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
     499      """Constructor.
     500
     501      Args:
     502        fd: io.Base or file object, The stream in which to write the downloaded
     503          bytes.
     504        request: googleapiclient.http.HttpRequest, the media request to perform in
     505          chunks.
     506        chunksize: int, File will be downloaded in chunks of this many bytes.
     507      """
     508      self._fd = fd
     509      self._request = request
     510      self._uri = request.uri
     511      self._chunksize = chunksize
     512      self._progress = 0
     513      self._total_size = None
     514      self._done = False
     515
     516      # Stubs for testing.
     517      self._sleep = time.sleep
     518      self._rand = random.random
     519
     520    @util.positional(1)
     521 -  def next_chunk(self, num_retries=0):
     522      """Get the next chunk of the download.
     523
     524      Args:
     525        num_retries: Integer, number of times to retry 500's with randomized
     526              exponential backoff. If all retries fail, the raised HttpError
     527              represents the last request. If zero (default), we attempt the
     528              request only once.
     529
     530      Returns:
     531        (status, done): (MediaDownloadStatus, boolean)
     532           The value of 'done' will be True when the media has been fully
     533           downloaded.
     534
     535      Raises:
     536        googleapiclient.errors.HttpError if the response was not a 2xx.
     537        httplib2.HttpLib2Error if a transport error has occured.
     538      """
     539      headers = {
     540          'range': 'bytes=%d-%d' % (
     541              self._progress, self._progress + self._chunksize)
     542          }
     543      http = self._request.http
     544
     545      for retry_num in range(num_retries + 1):
     546        if retry_num > 0:
     547          self._sleep(self._rand() * 2**retry_num)
     548          logging.warning(
     549              'Retry #%d for media download: GET %s, following status: %d'
     550              % (retry_num, self._uri, resp.status))
     551
     552        resp, content = http.request(self._uri, headers=headers)
     553        if resp.status < 500:
     554          break
     555
     556      if resp.status in [200, 206]:
     557        if 'content-location' in resp and resp['content-location'] != self._uri:
     558          self._uri = resp['content-location']
     559        self._progress += len(content)
     560        self._fd.write(content)
     561
     562        if 'content-range' in resp:
     563          content_range = resp['content-range']
     564          length = content_range.rsplit('/', 1)[1]
     565          self._total_size = int(length)
     566        elif 'content-length' in resp:
     567          self._total_size = int(resp['content-length'])
     568
     569        if self._progress == self._total_size:
     570          self._done = True
     571        return MediaDownloadProgress(self._progress, self._total_size), self._done
     572      else:
     573        raise HttpError(resp, content, uri=self._uri)
     574
     575
     576 -class _StreamSlice(object):
     577    """Truncated stream.
     578
     579    Takes a stream and presents a stream that is a slice of the original stream.
     580    This is used when uploading media in chunks. In later versions of Python a
     581    stream can be passed to httplib in place of the string of data to send. The
     582    problem is that httplib just blindly reads to the end of the stream. This
     583    wrapper presents a virtual stream that only reads to the end of the chunk.
     584    """
     585
     586 -  def __init__(self, stream, begin, chunksize):
     587      """Constructor.
     588
     589      Args:
     590        stream: (io.Base, file object), the stream to wrap.
     591        begin: int, the seek position the chunk begins at.
     592        chunksize: int, the size of the chunk.
     593      """
     594      self._stream = stream
     595      self._begin = begin
     596      self._chunksize = chunksize
     597      self._stream.seek(begin)
     598
     599 -  def read(self, n=-1):
     600      """Read n bytes.
     601
     602      Args:
     603        n, int, the number of bytes to read.
     604
     605      Returns:
     606        A string of length 'n', or less if EOF is reached.
     607      """
     608      # The data left available to read sits in [cur, end)
     609      cur = self._stream.tell()
     610      end = self._begin + self._chunksize
     611      if n == -1 or cur + n > end:
     612        n = end - cur
     613      return self._stream.read(n)
     614
     615
     616 -class HttpRequest(object):
     617    """Encapsulates a single HTTP request."""
     618
     619    @util.positional(4)
     620 -  def __init__(self, http, postproc, uri,
     621                 method='GET',
     622                 body=None,
     623                 headers=None,
     624                 methodId=None,
     625                 resumable=None):
     626      """Constructor for an HttpRequest.
     627
     628      Args:
     629        http: httplib2.Http, the transport object to use to make a request
     630        postproc: callable, called on the HTTP response and content to transform
     631                  it into a data object before returning, or raising an exception
     632                  on an error.
     633        uri: string, the absolute URI to send the request to
     634        method: string, the HTTP method to use
     635        body: string, the request body of the HTTP request,
     636        headers: dict, the HTTP request headers
     637        methodId: string, a unique identifier for the API method being called.
     638        resumable: MediaUpload, None if this is not a resumbale request.
     639      """
     640      self.uri = uri
     641      self.method = method
     642      self.body = body
     643      self.headers = headers or {}
     644      self.methodId = methodId
     645      self.http = http
     646      self.postproc = postproc
     647      self.resumable = resumable
     648      self.response_callbacks = []
     649      self._in_error_state = False
     650
     651      # Pull the multipart boundary out of the content-type header.
     652      major, minor, params = mimeparse.parse_mime_type(
     653          headers.get('content-type', 'application/json'))
     654
     655      # The size of the non-media part of the request.
     656      self.body_size = len(self.body or '')
     657
     658      # The resumable URI to send chunks to.
     659      self.resumable_uri = None
     660
     661      # The bytes that have been uploaded.
     662      self.resumable_progress = 0
     663
     664      # Stubs for testing.
     665      self._rand = random.random
     666      self._sleep = time.sleep
     667
     668    @util.positional(1)
     669 -  def execute(self, http=None, num_retries=0):
     670      """Execute the request.
     671
     672      Args:
     673        http: httplib2.Http, an http object to be used in place of the
     674              one the HttpRequest request object was constructed with.
     675        num_retries: Integer, number of times to retry 500's with randomized
     676              exponential backoff. If all retries fail, the raised HttpError
     677              represents the last request. If zero (default), we attempt the
     678              request only once.
     679
     680      Returns:
     681        A deserialized object model of the response body as determined
     682        by the postproc.
     683
     684      Raises:
     685        googleapiclient.errors.HttpError if the response was not a 2xx.
     686        httplib2.HttpLib2Error if a transport error has occured.
     687      """
     688      if http is None:
     689        http = self.http
     690
     691      if self.resumable:
     692        body = None
     693        while body is None:
     694          _, body = self.next_chunk(http=http, num_retries=num_retries)
     695        return body
     696
     697      # Non-resumable case.
     698
     699      if 'content-length' not in self.headers:
     700        self.headers['content-length'] = str(self.body_size)
     701      # If the request URI is too long then turn it into a POST request.
     702      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
     703        self.method = 'POST'
     704        self.headers['x-http-method-override'] = 'GET'
     705        self.headers['content-type'] = 'application/x-www-form-urlencoded'
     706        parsed = urlparse(self.uri)
     707        self.uri = urlunparse(
     708            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
     709             None)
     710            )
     711        self.body = parsed.query
     712        self.headers['content-length'] = str(len(self.body))
     713
     714      # Handle retries for server-side errors.
     715      for retry_num in range(num_retries + 1):
     716        if retry_num > 0:
     717          self._sleep(self._rand() * 2**retry_num)
     718          logging.warning('Retry #%d for request: %s %s, following status: %d'
     719                          % (retry_num, self.method, self.uri, resp.status))
     720
     721        resp, content = http.request(str(self.uri), method=str(self.method),
     722                                     body=self.body, headers=self.headers)
     723        if resp.status < 500:
     724          break
     725
     726      for callback in self.response_callbacks:
     727        callback(resp)
     728      if resp.status >= 300:
     729        raise HttpError(resp, content, uri=self.uri)
     730      return self.postproc(resp, content)
     731
     732    @util.positional(2)
     733 -  def add_response_callback(self, cb):
     734      """add_response_headers_callback
     735
     736      Args:
     737        cb: Callback to be called on receiving the response headers, of signature:
     738
     739        def cb(resp):
     740          # Where resp is an instance of httplib2.Response
     741      """
     742      self.response_callbacks.append(cb)
     743
     744    @util.positional(1)
     745 -  def next_chunk(self, http=None, num_retries=0):
     746      """Execute the next step of a resumable upload.
     747
     748      Can only be used if the method being executed supports media uploads and
     749      the MediaUpload object passed in was flagged as using resumable upload.
     750
     751      Example:
     752
     753        media = MediaFileUpload('cow.png', mimetype='image/png',
     754                                chunksize=1000, resumable=True)
     755        request = farm.animals().insert(
     756            id='cow',
     757            name='cow.png',
     758            media_body=media)
     759
     760        response = None
     761        while response is None:
     762          status, response = request.next_chunk()
     763          if status:
     764            print "Upload %d%% complete." % int(status.progress() * 100)
     765
     766
     767      Args:
     768        http: httplib2.Http, an http object to be used in place of the
     769              one the HttpRequest request object was constructed with.
     770        num_retries: Integer, number of times to retry 500's with randomized
     771              exponential backoff. If all retries fail, the raised HttpError
     772              represents the last request. If zero (default), we attempt the
     773              request only once.
     774
     775      Returns:
     776        (status, body): (ResumableMediaStatus, object)
     777           The body will be None until the resumable media is fully uploaded.
     778
     779      Raises:
     780        googleapiclient.errors.HttpError if the response was not a 2xx.
     781        httplib2.HttpLib2Error if a transport error has occured.
     782      """
     783      if http is None:
     784        http = self.http
     785
     786      if self.resumable.size() is None:
     787        size = '*'
     788      else:
     789        size = str(self.resumable.size())
     790
     791      if self.resumable_uri is None:
     792        start_headers = copy.copy(self.headers)
     793        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
     794        if size != '*':
     795          start_headers['X-Upload-Content-Length'] = size
     796        start_headers['content-length'] = str(self.body_size)
     797
     798        for retry_num in range(num_retries + 1):
     799          if retry_num > 0:
     800            self._sleep(self._rand() * 2**retry_num)
     801            logging.warning(
     802                'Retry #%d for resumable URI request: %s %s, following status: %d'
     803                % (retry_num, self.method, self.uri, resp.status))
     804
     805          resp, content = http.request(self.uri, method=self.method,
     806                                       body=self.body,
     807                                       headers=start_headers)
     808          if resp.status < 500:
     809            break
     810
     811        if resp.status == 200 and 'location' in resp:
     812          self.resumable_uri = resp['location']
     813        else:
     814          raise ResumableUploadError(resp, content)
     815      elif self._in_error_state:
     816        # If we are in an error state then query the server for current state of
     817        # the upload by sending an empty PUT and reading the 'range' header in
     818        # the response.
     819        headers = {
     820            'Content-Range': 'bytes */%s' % size,
     821            'content-length': '0'
     822            }
     823        resp, content = http.request(self.resumable_uri, 'PUT',
     824                                     headers=headers)
     825        status, body = self._process_response(resp, content)
     826        if body:
     827          # The upload was complete.
     828          return (status, body)
     829
     830      # The httplib.request method can take streams for the body parameter, but
     831      # only in Python 2.6 or later. If a stream is available under those
     832      # conditions then use it as the body argument.
     833      if self.resumable.has_stream() and sys.version_info[1] >= 6:
     834        data = self.resumable.stream()
     835        if self.resumable.chunksize() == -1:
     836          data.seek(self.resumable_progress)
     837          chunk_end = self.resumable.size() - self.resumable_progress - 1
     838        else:
     839          # Doing chunking with a stream, so wrap a slice of the stream.
     840          data = _StreamSlice(data, self.resumable_progress,
     841                              self.resumable.chunksize())
     842          chunk_end = min(
     843              self.resumable_progress + self.resumable.chunksize() - 1,
     844              self.resumable.size() - 1)
     845      else:
     846        data = self.resumable.getbytes(
     847            self.resumable_progress, self.resumable.chunksize())
     848
     849        # A short read implies that we are at EOF, so finish the upload.
     850        if len(data) < self.resumable.chunksize():
     851          size = str(self.resumable_progress + len(data))
     852
     853        chunk_end = self.resumable_progress + len(data) - 1
     854
     855      headers = {
     856          'Content-Range': 'bytes %d-%d/%s' % (
     857              self.resumable_progress, chunk_end, size),
     858          # Must set the content-length header here because httplib can't
     859          # calculate the size when working with _StreamSlice.
     860          'Content-Length': str(chunk_end - self.resumable_progress + 1)
     861          }
     862
     863      for retry_num in range(num_retries + 1):
     864        if retry_num > 0:
     865          self._sleep(self._rand() * 2**retry_num)
     866          logging.warning(
     867              'Retry #%d for media upload: %s %s, following status: %d'
     868              % (retry_num, self.method, self.uri, resp.status))
     869
     870        try:
     871          resp, content = http.request(self.resumable_uri, method='PUT',
     872                                       body=data,
     873                                       headers=headers)
     874        except:
     875          self._in_error_state = True
     876          raise
     877        if resp.status < 500:
     878          break
     879
     880      return self._process_response(resp, content)
     881
     882 -  def _process_response(self, resp, content):
     883      """Process the response from a single chunk upload.
     884
     885      Args:
     886        resp: httplib2.Response, the response object.
     887        content: string, the content of the response.
     888
     889      Returns:
     890        (status, body): (ResumableMediaStatus, object)
     891           The body will be None until the resumable media is fully uploaded.
     892
     893      Raises:
     894        googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
     895      """
     896      if resp.status in [200, 201]:
     897        self._in_error_state = False
     898        return None, self.postproc(resp, content)
     899      elif resp.status == 308:
     900        self._in_error_state = False
     901        # A "308 Resume Incomplete" indicates we are not done.
     902        self.resumable_progress = int(resp['range'].split('-')[1]) + 1
     903        if 'location' in resp:
     904          self.resumable_uri = resp['location']
     905      else:
     906        self._in_error_state = True
     907        raise HttpError(resp, content, uri=self.uri)
     908
     909      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
     910              None)
     911
     912 -  def to_json(self):
     913      """Returns a JSON representation of the HttpRequest."""
     914      d = copy.copy(self.__dict__)
     915      if d['resumable'] is not None:
     916        d['resumable'] = self.resumable.to_json()
     917      del d['http']
     918      del d['postproc']
     919      del d['_sleep']
     920      del d['_rand']
     921
     922      return json.dumps(d)
     923
     924    @staticmethod
     925 -  def from_json(s, http, postproc):
     926      """Returns an HttpRequest populated with info from a JSON object."""
     927      d = json.loads(s)
     928      if d['resumable'] is not None:
     929        d['resumable'] = MediaUpload.new_from_json(d['resumable'])
     930      return HttpRequest(
     931          http,
     932          postproc,
     933          uri=d['uri'],
     934          method=d['method'],
     935          body=d['body'],
     936          headers=d['headers'],
     937          methodId=d['methodId'],
     938          resumable=d['resumable'])
     939
     940
     941 -class BatchHttpRequest(object):
     942    """Batches multiple HttpRequest objects into a single HTTP request.
     943
     944    Example:
     945      from googleapiclient.http import BatchHttpRequest
     946
     947      def list_animals(request_id, response, exception):
     948        \"\"\"Do something with the animals list response.\"\"\"
     949        if exception is not None:
     950          # Do something with the exception.
     951          pass
     952        else:
     953          # Do something with the response.
     954          pass
     955
     956      def list_farmers(request_id, response, exception):
     957        \"\"\"Do something with the farmers list response.\"\"\"
     958        if exception is not None:
     959          # Do something with the exception.
     960          pass
     961        else:
     962          # Do something with the response.
     963          pass
     964
     965      service = build('farm', 'v2')
     966
     967      batch = BatchHttpRequest()
     968
     969      batch.add(service.animals().list(), list_animals)
     970      batch.add(service.farmers().list(), list_farmers)
     971      batch.execute(http=http)
     972    """
     973
     974    @util.positional(1)
     975 -  def __init__(self, callback=None, batch_uri=None):
     976      """Constructor for a BatchHttpRequest.
     977
     978      Args:
     979        callback: callable, A callback to be called for each response, of the
     980          form callback(id, response, exception). The first parameter is the
     981          request id, and the second is the deserialized response object. The
     982          third is an googleapiclient.errors.HttpError exception object if an HTTP error
     983          occurred while processing the request, or None if no error occurred.
     984        batch_uri: string, URI to send batch requests to.
     985      """
     986      if batch_uri is None:
     987        batch_uri = 'https://www.googleapis.com/batch'
     988      self._batch_uri = batch_uri
     989
     990      # Global callback to be called for each individual response in the batch.
     991      self._callback = callback
     992
     993      # A map from id to request.
     994      self._requests = {}
     995
     996      # A map from id to callback.
     997      self._callbacks = {}
     998
     999      # List of request ids, in the order in which they were added.
    1000      self._order = []
    1001
    1002      # The last auto generated id.
    1003      self._last_auto_id = 0
    1004
    1005      # Unique ID on which to base the Content-ID headers.
    1006      self._base_id = None
    1007
    1008      # A map from request id to (httplib2.Response, content) response pairs
    1009      self._responses = {}
    1010
    1011      # A map of id(Credentials) that have been refreshed.
    1012      self._refreshed_credentials = {}
    1013
    1014 -  def _refresh_and_apply_credentials(self, request, http):
    1015      """Refresh the credentials and apply to the request.
    1016
    1017      Args:
    1018        request: HttpRequest, the request.
    1019        http: httplib2.Http, the global http object for the batch.
    1020      """
    1021      # For the credentials to refresh, but only once per refresh_token
    1022      # If there is no http per the request then refresh the http passed in
    1023      # via execute()
    1024      creds = None
    1025      if request.http is not None and hasattr(request.http.request,
    1026          'credentials'):
    1027        creds = request.http.request.credentials
    1028      elif http is not None and hasattr(http.request, 'credentials'):
    1029        creds = http.request.credentials
    1030      if creds is not None:
    1031        if id(creds) not in self._refreshed_credentials:
    1032          creds.refresh(http)
    1033          self._refreshed_credentials[id(creds)] = 1
    1034
    1035      # Only apply the credentials if we are using the http object passed in,
    1036      # otherwise apply() will get called during _serialize_request().
    1037      if request.http is None or not hasattr(request.http.request,
    1038          'credentials'):
    1039        creds.apply(request.headers)
    1040
    1041 -  def _id_to_header(self, id_):
    1042      """Convert an id to a Content-ID header value.
    1043
    1044      Args:
    1045        id_: string, identifier of individual request.
    1046
    1047      Returns:
    1048        A Content-ID header with the id_ encoded into it. A UUID is prepended to
    1049        the value because Content-ID headers are supposed to be universally
    1050        unique.
    1051      """
    1052      if self._base_id is None:
    1053        self._base_id = uuid.uuid4()
    1054
    1055      return '<%s+%s>' % (self._base_id, quote(id_))
    1056
    1057 -  def _header_to_id(self, header):
    1058      """Convert a Content-ID header value to an id.
    1059
    1060      Presumes the Content-ID header conforms to the format that _id_to_header()
    1061      returns.
    1062
    1063      Args:
    1064        header: string, Content-ID header value.
    1065
    1066      Returns:
    1067        The extracted id value.
    1068
    1069      Raises:
    1070        BatchError if the header is not in the expected format.
    1071      """
    1072      if header[0] != '<' or header[-1] != '>':
    1073        raise BatchError("Invalid value for Content-ID: %s" % header)
    1074      if '+' not in header:
    1075        raise BatchError("Invalid value for Content-ID: %s" % header)
    1076      base, id_ = header[1:-1].rsplit('+', 1)
    1077
    1078      return unquote(id_)
    1079
    1080 -  def _serialize_request(self, request):
    1081      """Convert an HttpRequest object into a string.
    1082
    1083      Args:
    1084        request: HttpRequest, the request to serialize.
    1085
    1086      Returns:
    1087        The request as a string in application/http format.
    1088      """
    1089      # Construct status line
    1090      parsed = urlparse(request.uri)
    1091      request_line = urlunparse(
    1092          ('', '', parsed.path, parsed.params, parsed.query, '')
    1093          )
    1094      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
    1095      major, minor = request.headers.get('content-type', 'application/json').split('/')
    1096      msg = MIMENonMultipart(major, minor)
    1097      headers = request.headers.copy()
    1098
    1099      if request.http is not None and hasattr(request.http.request,
    1100          'credentials'):
    1101        request.http.request.credentials.apply(headers)
    1102
    1103      # MIMENonMultipart adds its own Content-Type header.
    1104      if 'content-type' in headers:
    1105        del headers['content-type']
    1106
    1107      for key, value in six.iteritems(headers):
    1108        msg[key] = value
    1109      msg['Host'] = parsed.netloc
    1110      msg.set_unixfrom(None)
    1111
    1112      if request.body is not None:
    1113        msg.set_payload(request.body)
    1114        msg['content-length'] = str(len(request.body))
    1115
    1116      # Serialize the mime message.
    1117      fp = StringIO()
    1118      # maxheaderlen=0 means don't line wrap headers.
    1119      g = Generator(fp, maxheaderlen=0)
    1120      g.flatten(msg, unixfrom=False)
    1121      body = fp.getvalue()
    1122
    1123      return status_line + body
    1124
    1125 -  def _deserialize_response(self, payload):
    1126      """Convert string into httplib2 response and content.
    1127
    1128      Args:
    1129        payload: string, headers and body as a string.
    1130
    1131      Returns:
    1132        A pair (resp, content), such as would be returned from httplib2.request.
    1133      """
    1134      # Strip off the status line
    1135      status_line, payload = payload.split('\n', 1)
    1136      protocol, status, reason = status_line.split(' ', 2)
    1137
    1138      # Parse the rest of the response
    1139      parser = FeedParser()
    1140      parser.feed(payload)
    1141      msg = parser.close()
    1142      msg['status'] = status
    1143
    1144      # Create httplib2.Response from the parsed headers.
    1145      resp = httplib2.Response(msg)
    1146      resp.reason = reason
    1147      resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
    1148
    1149      content = payload.split('\r\n\r\n', 1)[1]
    1150
    1151      return resp, content
    1152
    1153 -  def _new_id(self):
    1154      """Create a new id.
    1155
    1156      Auto incrementing number that avoids conflicts with ids already used.
    1157
    1158      Returns:
    1159         string, a new unique id.
    1160      """
    1161      self._last_auto_id += 1
    1162      while str(self._last_auto_id) in self._requests:
    1163        self._last_auto_id += 1
    1164      return str(self._last_auto_id)
    1165
    1166    @util.positional(2)
    1167 -  def add(self, request, callback=None, request_id=None):
    1168      """Add a new request.
    1169
    1170      Every callback added will be paired with a unique id, the request_id. That
    1171      unique id will be passed back to the callback when the response comes back
    1172      from the server. The default behavior is to have the library generate it's
    1173      own unique id. If the caller passes in a request_id then they must ensure
    1174      uniqueness for each request_id, and if they are not an exception is
    1175      raised. Callers should either supply all request_ids or nevery supply a
    1176      request id, to avoid such an error.
    1177
    1178      Args:
    1179        request: HttpRequest, Request to add to the batch.
    1180        callback: callable, A callback to be called for this response, of the
    1181          form callback(id, response, exception). The first parameter is the
    1182          request id, and the second is the deserialized response object. The
    1183          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1184          occurred while processing the request, or None if no errors occurred.
    1185        request_id: string, A unique id for the request. The id will be passed to
    1186          the callback with the response.
    1187
    1188      Returns:
    1189        None
    1190
    1191      Raises:
    1192        BatchError if a media request is added to a batch.
    1193        KeyError is the request_id is not unique.
    1194      """
    1195      if request_id is None:
    1196        request_id = self._new_id()
    1197      if request.resumable is not None:
    1198        raise BatchError("Media requests cannot be used in a batch request.")
    1199      if request_id in self._requests:
    1200        raise KeyError("A request with this ID already exists: %s" % request_id)
    1201      self._requests[request_id] = request
    1202      self._callbacks[request_id] = callback
    1203      self._order.append(request_id)
    1204
    1205 -  def _execute(self, http, order, requests):
    1206      """Serialize batch request, send to server, process response.
    1207
    1208      Args:
    1209        http: httplib2.Http, an http object to be used to make the request with.
    1210        order: list, list of request ids in the order they were added to the
    1211          batch.
    1212        request: list, list of request objects to send.
    1213
    1214      Raises:
    1215        httplib2.HttpLib2Error if a transport error has occured.
    1216        googleapiclient.errors.BatchError if the response is the wrong format.
    1217      """
    1218      message = MIMEMultipart('mixed')
    1219      # Message should not write out it's own headers.
    1220      setattr(message, '_write_headers', lambda self: None)
    1221
    1222      # Add all the individual requests.
    1223      for request_id in order:
    1224        request = requests[request_id]
    1225
    1226        msg = MIMENonMultipart('application', 'http')
    1227        msg['Content-Transfer-Encoding'] = 'binary'
    1228        msg['Content-ID'] = self._id_to_header(request_id)
    1229
    1230        body = self._serialize_request(request)
    1231        msg.set_payload(body)
    1232        message.attach(msg)
    1233
    1234      # encode the body: note that we can't use `as_string`, because
    1235      # it plays games with `From ` lines.
    1236      fp = StringIO()
    1237      g = Generator(fp, mangle_from_=False)
    1238      g.flatten(message, unixfrom=False)
    1239      body = fp.getvalue()
    1240
    1241      headers = {}
    1242      headers['content-type'] = ('multipart/mixed; '
    1243                                 'boundary="%s"') % message.get_boundary()
    1244
    1245      resp, content = http.request(self._batch_uri, method='POST', body=body,
    1246                                   headers=headers)
    1247
    1248      if resp.status >= 300:
    1249        raise HttpError(resp, content, uri=self._batch_uri)
    1250
    1251      # Prepend with a content-type header so FeedParser can handle it.
    1252      header = 'content-type: %s\r\n\r\n' % resp['content-type']
    1253      # PY3's FeedParser only accepts unicode. So we should decode content
    1254      # here, and encode each payload again.
    1255      if six.PY3:
    1256        content = content.decode('utf-8')
    1257      for_parser = header + content
    1258
    1259      parser = FeedParser()
    1260      parser.feed(for_parser)
    1261      mime_response = parser.close()
    1262
    1263      if not mime_response.is_multipart():
    1264        raise BatchError("Response not in multipart/mixed format.", resp=resp,
    1265                         content=content)
    1266
    1267      for part in mime_response.get_payload():
    1268        request_id = self._header_to_id(part['Content-ID'])
    1269        response, content = self._deserialize_response(part.get_payload())
    1270        # We encode content here to emulate normal http response.
    1271        if isinstance(content, six.text_type):
    1272          content = content.encode('utf-8')
    1273        self._responses[request_id] = (response, content)
    1274
    1275    @util.positional(1)
    1276 -  def execute(self, http=None):
    1277      """Execute all the requests as a single batched HTTP request.
    1278
    1279      Args:
    1280        http: httplib2.Http, an http object to be used in place of the one the
    1281          HttpRequest request object was constructed with. If one isn't supplied
    1282          then use a http object from the requests in this batch.
    1283
    1284      Returns:
    1285        None
    1286
    1287      Raises:
    1288        httplib2.HttpLib2Error if a transport error has occured.
    1289        googleapiclient.errors.BatchError if the response is the wrong format.
    1290      """
    1291      # If we have no requests return
    1292      if len(self._order) == 0:
    1293        return None
    1294
    1295      # If http is not supplied use the first valid one given in the requests.
    1296      if http is None:
    1297        for request_id in self._order:
    1298          request = self._requests[request_id]
    1299          if request is not None:
    1300            http = request.http
    1301            break
    1302
    1303      if http is None:
    1304        raise ValueError("Missing a valid http object.")
    1305
    1306      self._execute(http, self._order, self._requests)
    1307
    1308      # Loop over all the requests and check for 401s. For each 401 request the
    1309      # credentials should be refreshed and then sent again in a separate batch.
    1310      redo_requests = {}
    1311      redo_order = []
    1312
    1313      for request_id in self._order:
    1314        resp, content = self._responses[request_id]
    1315        if resp['status'] == '401':
    1316          redo_order.append(request_id)
    1317          request = self._requests[request_id]
    1318          self._refresh_and_apply_credentials(request, http)
    1319          redo_requests[request_id] = request
    1320
    1321      if redo_requests:
    1322        self._execute(http, redo_order, redo_requests)
    1323
    1324      # Now process all callbacks that are erroring, and raise an exception for
    1325      # ones that return a non-2xx response? Or add extra parameter to callback
    1326      # that contains an HttpError?
    1327
    1328      for request_id in self._order:
    1329        resp, content = self._responses[request_id]
    1330
    1331        request = self._requests[request_id]
    1332        callback = self._callbacks[request_id]
    1333
    1334        response = None
    1335        exception = None
    1336        try:
    1337          if resp.status >= 300:
    1338            raise HttpError(resp, content, uri=request.uri)
    1339          response = request.postproc(resp, content)
    1340        except HttpError as e:
    1341          exception = e
    1342
    1343        if callback is not None:
    1344          callback(request_id, response, exception)
    1345        if self._callback is not None:
    1346          self._callback(request_id, response, exception)
    1347
    1348
    1349 -class HttpRequestMock(object):
    1350    """Mock of HttpRequest.
    1351
    1352    Do not construct directly, instead use RequestMockBuilder.
    1353    """
    1354
    1355 -  def __init__(self, resp, content, postproc):
    1356      """Constructor for HttpRequestMock
    1357
    1358      Args:
    1359        resp: httplib2.Response, the response to emulate coming from the request
    1360        content: string, the response body
    1361        postproc: callable, the post processing function usually supplied by
    1362                  the model class. See model.JsonModel.response() as an example.
    1363      """
    1364      self.resp = resp
    1365      self.content = content
    1366      self.postproc = postproc
    1367      if resp is None:
    1368        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
    1369      if 'reason' in self.resp:
    1370        self.resp.reason = self.resp['reason']
    1371
    1372 -  def execute(self, http=None):
    1373      """Execute the request.
    1374
    1375      Same behavior as HttpRequest.execute(), but the response is
    1376      mocked and not really from an HTTP request/response.
    1377      """
    1378      return self.postproc(self.resp, self.content)
    1379
    1380
    1381 -class RequestMockBuilder(object):
    1382    """A simple mock of HttpRequest
    1383
    1384      Pass in a dictionary to the constructor that maps request methodIds to
    1385      tuples of (httplib2.Response, content, opt_expected_body) that should be
    1386      returned when that method is called. None may also be passed in for the
    1387      httplib2.Response, in which case a 200 OK response will be generated.
    1388      If an opt_expected_body (str or dict) is provided, it will be compared to
    1389      the body and UnexpectedBodyError will be raised on inequality.
    1390
    1391      Example:
    1392        response = '{"data": {"id": "tag:google.c...'
    1393        requestBuilder = RequestMockBuilder(
    1394          {
    1395            'plus.activities.get': (None, response),
    1396          }
    1397        )
    1398        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
    1399
    1400      Methods that you do not supply a response for will return a
    1401      200 OK with an empty string as the response content or raise an excpetion
    1402      if check_unexpected is set to True. The methodId is taken from the rpcName
    1403      in the discovery document.
    1404
    1405      For more details see the project wiki.
    1406    """
    1407
    1408 -  def __init__(self, responses, check_unexpected=False):
    1409      """Constructor for RequestMockBuilder
    1410
    1411      The constructed object should be a callable object
    1412      that can replace the class HttpResponse.
    1413
    1414      responses - A dictionary that maps methodIds into tuples
    1415                  of (httplib2.Response, content). The methodId
    1416                  comes from the 'rpcName' field in the discovery
    1417                  document.
    1418      check_unexpected - A boolean setting whether or not UnexpectedMethodError
    1419                         should be raised on unsupplied method.
    1420      """
    1421      self.responses = responses
    1422      self.check_unexpected = check_unexpected
    1423
    1424 -  def __call__(self, http, postproc, uri, method='GET', body=None,
    1425                 headers=None, methodId=None, resumable=None):
    1426      """Implements the callable interface that discovery.build() expects
    1427      of requestBuilder, which is to build an object compatible with
    1428      HttpRequest.execute(). See that method for the description of the
    1429      parameters and the expected response.
    1430      """
    1431      if methodId in self.responses:
    1432        response = self.responses[methodId]
    1433        resp, content = response[:2]
    1434        if len(response) > 2:
    1435          # Test the body against the supplied expected_body.
    1436          expected_body = response[2]
    1437          if bool(expected_body) != bool(body):
    1438            # Not expecting a body and provided one
    1439            # or expecting a body and not provided one.
    1440            raise UnexpectedBodyError(expected_body, body)
    1441          if isinstance(expected_body, str):
    1442            expected_body = json.loads(expected_body)
    1443          body = json.loads(body)
    1444          if body != expected_body:
    1445            raise UnexpectedBodyError(expected_body, body)
    1446        return HttpRequestMock(resp, content, postproc)
    1447      elif self.check_unexpected:
    1448        raise UnexpectedMethodError(methodId=methodId)
    1449      else:
    1450        model = JsonModel(False)
    1451        return HttpRequestMock(None, '{}', model.response)
    1452
    1453
    1454 -class HttpMock(object):
    1455    """Mock of httplib2.Http"""
    1456
    1457 -  def __init__(self, filename=None, headers=None):
    1458      """
    1459      Args:
    1460        filename: string, absolute filename to read response from
    1461        headers: dict, header to return with response
    1462      """
    1463      if headers is None:
    1464        headers = {'status': '200'}
    1465      if filename:
    1466        f = open(filename, 'rb')
    1467        self.data = f.read()
    1468        f.close()
    1469      else:
    1470        self.data = None
    1471      self.response_headers = headers
    1472      self.headers = None
    1473      self.uri = None
    1474      self.method = None
    1475      self.body = None
    1476      self.headers = None
    1477
    1478
    1479 -  def request(self, uri,
    1480                method='GET',
    1481                body=None,
    1482                headers=None,
    1483                redirections=1,
    1484                connection_type=None):
    1485      self.uri = uri
    1486      self.method = method
    1487      self.body = body
    1488      self.headers = headers
    1489      return httplib2.Response(self.response_headers), self.data
    1490
    1491
    1492 -class HttpMockSequence(object):
    1493    """Mock of httplib2.Http
    1494
    1495    Mocks a sequence of calls to request returning different responses for each
    1496    call. Create an instance initialized with the desired response headers
    1497    and content and then use as if an httplib2.Http instance.
    1498
    1499      http = HttpMockSequence([
    1500        ({'status': '401'}, ''),
    1501        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
    1502        ({'status': '200'}, 'echo_request_headers'),
    1503        ])
    1504      resp, content = http.request("http://examples.com")
    1505
    1506    There are special values you can pass in for content to trigger
    1507    behavours that are helpful in testing.
    1508
    1509    'echo_request_headers' means return the request headers in the response body
    1510    'echo_request_headers_as_json' means return the request headers in
    1511       the response body
    1512    'echo_request_body' means return the request body in the response body
    1513    'echo_request_uri' means return the request uri in the response body
    1514    """
    1515
    1516 -  def __init__(self, iterable):
    1517      """
    1518      Args:
    1519        iterable: iterable, a sequence of pairs of (headers, body)
    1520      """
    1521      self._iterable = iterable
    1522      self.follow_redirects = True
    1523
    1524 -  def request(self, uri,
    1525                method='GET',
    1526                body=None,
    1527                headers=None,
    1528                redirections=1,
    1529                connection_type=None):
    1530      resp, content = self._iterable.pop(0)
    1531      if content == 'echo_request_headers':
    1532        content = headers
    1533      elif content == 'echo_request_headers_as_json':
    1534        content = json.dumps(headers)
    1535      elif content == 'echo_request_body':
    1536        if hasattr(body, 'read'):
    1537          content = body.read()
    1538        else:
    1539          content = body
    1540      elif content == 'echo_request_uri':
    1541        content = uri
    1542      if isinstance(content, six.text_type):
    1543        content = content.encode('utf-8')
    1544      return httplib2.Response(resp), content
    1545
    1546
    1547 -def set_user_agent(http, user_agent):
    1548    """Set the user-agent on every request.
    1549
    1550    Args:
    1551       http - An instance of httplib2.Http
    1552           or something that acts like it.
    1553       user_agent: string, the value for the user-agent header.
    1554
    1555    Returns:
    1556       A modified instance of http that was passed in.
    1557
    1558    Example:
    1559
    1560      h = httplib2.Http()
    1561      h = set_user_agent(h, "my-app-name/6.0")
    1562
    1563    Most of the time the user-agent will be set doing auth, this is for the rare
    1564    cases where you are accessing an unauthenticated endpoint.
    1565    """
    1566    request_orig = http.request
    1567
    1568    # The closure that will replace 'httplib2.Http.request'.
    1569    def new_request(uri, method='GET', body=None, headers=None,
    1570                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1571                    connection_type=None):
    1572      """Modify the request headers to add the user-agent."""
    1573      if headers is None:
    1574        headers = {}
    1575      if 'user-agent' in headers:
    1576        headers['user-agent'] = user_agent + ' ' + headers['user-agent']
    1577      else:
    1578        headers['user-agent'] = user_agent
    1579      resp, content = request_orig(uri, method, body, headers,
    1580                          redirections, connection_type)
    1581      return resp, content
    1582
    1583    http.request = new_request
    1584    return http
    1585
    1586
    1587 -def tunnel_patch(http):
    1588    """Tunnel PATCH requests over POST.
    1589    Args:
    1590       http - An instance of httplib2.Http
    1591           or something that acts like it.
    1592
    1593    Returns:
    1594       A modified instance of http that was passed in.
    1595
    1596    Example:
    1597
    1598      h = httplib2.Http()
    1599      h = tunnel_patch(h, "my-app-name/6.0")
    1600
    1601    Useful if you are running on a platform that doesn't support PATCH.
    1602    Apply this last if you are using OAuth 1.0, as changing the method
    1603    will result in a different signature.
    1604    """
    1605    request_orig = http.request
    1606
    1607    # The closure that will replace 'httplib2.Http.request'.
    1608    def new_request(uri, method='GET', body=None, headers=None,
    1609                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1610                    connection_type=None):
    1611      """Modify the request headers to add the user-agent."""
    1612      if headers is None:
    1613        headers = {}
    1614      if method == 'PATCH':
    1615        if 'oauth_token' in headers.get('authorization', ''):
    1616          logging.warning(
    1617              'OAuth 1.0 request made with Credentials after tunnel_patch.')
    1618        headers['x-http-method-override'] = "PATCH"
    1619        method = 'POST'
    1620      resp, content = request_orig(uri, method, body, headers,
    1621                          redirections, connection_type)
    1622      return resp, content
    1623
    1624    http.request = new_request
    1625    return http
    1626