1   
   2   
   3   
   4   
   5   
   6   
   7   
   8   
   9   
  10   
  11   
  12   
  13   
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actual HTTP request. 
  20  """ 
  21  from __future__ import absolute_import 
  22  import six 
  23  from six.moves import http_client 
  24  from six.moves import range 
  25   
  26  __author__ = 'jcgregorio@google.com (Joe Gregorio)' 
  27   
  28  from six import BytesIO, StringIO 
  29  from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote 
  30   
  31  import base64 
  32  import copy 
  33  import gzip 
  34  import httplib2 
  35  import json 
  36  import logging 
  37  import mimetypes 
  38  import os 
  39  import random 
  40  import socket 
  41  import sys 
  42  import time 
  43  import uuid 
  44   
  45   
  46  try: 
  47    import ssl 
  48  except ImportError: 
  49    _ssl_SSLError = object() 
  50  else: 
  51    _ssl_SSLError = ssl.SSLError 
  52   
  53  from email.generator import Generator 
  54  from email.mime.multipart import MIMEMultipart 
  55  from email.mime.nonmultipart import MIMENonMultipart 
  56  from email.parser import FeedParser 
  57   
  58  from googleapiclient import _helpers as util 
  59   
  60  from googleapiclient import _auth 
  61  from googleapiclient.errors import BatchError 
  62  from googleapiclient.errors import HttpError 
  63  from googleapiclient.errors import InvalidChunkSizeError 
  64  from googleapiclient.errors import ResumableUploadError 
  65  from googleapiclient.errors import UnexpectedBodyError 
  66  from googleapiclient.errors import UnexpectedMethodError 
  67  from googleapiclient.model import JsonModel 
  68   
  69   
  70  LOGGER = logging.getLogger(__name__) 
  71   
  72  DEFAULT_CHUNK_SIZE = 100*1024*1024 
  73   
  74  MAX_URI_LENGTH = 2048 
  75   
  76  MAX_BATCH_LIMIT = 1000 
  77   
  78  _TOO_MANY_REQUESTS = 429 
  79   
  80  DEFAULT_HTTP_TIMEOUT_SEC = 60 
  81   
  82  _LEGACY_BATCH_URI = 'https://www.googleapis.com/batch' 
  86    """Determines whether a response should be retried. 
  87   
  88    Args: 
  89      resp_status: The response status received. 
  90      content: The response content body. 
  91   
  92    Returns: 
  93      True if the response should be retried, otherwise False. 
  94    """ 
  95     
  96    if resp_status >= 500: 
  97      return True 
  98   
  99     
 100    if resp_status == _TOO_MANY_REQUESTS: 
 101      return True 
 102   
 103     
 104     
 105    if resp_status == six.moves.http_client.FORBIDDEN: 
 106       
 107      if not content: 
 108        return False 
 109   
 110       
 111      try: 
 112        data = json.loads(content.decode('utf-8')) 
 113        if isinstance(data, dict): 
 114          reason = data['error']['errors'][0]['reason'] 
 115        else: 
 116          reason = data[0]['error']['errors']['reason'] 
 117      except (UnicodeDecodeError, ValueError, KeyError): 
 118        LOGGER.warning('Invalid JSON content from response: %s', content) 
 119        return False 
 120   
 121      LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) 
 122   
 123       
 124      if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ): 
 125        return True 
 126   
 127     
 128    return False 
  129   
 130   
 131 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, 
 132                     **kwargs): 
  133    """Retries an HTTP request multiple times while handling errors. 
 134   
 135    If after all retries the request still fails, last error is either returned as 
 136    return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). 
 137   
 138    Args: 
 139      http: Http object to be used to execute request. 
 140      num_retries: Maximum number of retries. 
 141      req_type: Type of the request (used for logging retries). 
 142      sleep, rand: Functions to sleep for random time between retries. 
 143      uri: URI to be requested. 
 144      method: HTTP method to be used. 
 145      args, kwargs: Additional arguments passed to http.request. 
 146   
 147    Returns: 
 148      resp, content - Response from the http request (may be HTTP 5xx). 
 149    """ 
 150    resp = None 
 151    content = None 
 152    for retry_num in range(num_retries + 1): 
 153      if retry_num > 0: 
 154         
 155        sleep_time = rand() * 2 ** retry_num 
 156        LOGGER.warning( 
 157            'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s', 
 158            sleep_time, retry_num, num_retries, req_type, method, uri, 
 159            resp.status if resp else exception) 
 160        sleep(sleep_time) 
 161   
 162      try: 
 163        exception = None 
 164        resp, content = http.request(uri, method, *args, **kwargs) 
 165       
 166      except _ssl_SSLError as ssl_error: 
 167        exception = ssl_error 
 168      except socket.timeout as socket_timeout: 
 169         
 170         
 171        exception = socket_timeout 
 172      except socket.error as socket_error: 
 173         
 174        if socket.errno.errorcode.get(socket_error.errno) not in { 
 175          'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED'}: 
 176          raise 
 177        exception = socket_error 
 178      except httplib2.ServerNotFoundError as server_not_found_error: 
 179        exception = server_not_found_error 
 180   
 181      if exception: 
 182        if retry_num == num_retries: 
 183          raise exception 
 184        else: 
 185          continue 
 186   
 187      if not _should_retry_response(resp.status, content): 
 188        break 
 189   
 190    return resp, content 
  191   
 218   
 244   
 387   
 512   
 578   
 607   
 706   
 709    """Truncated stream. 
 710   
 711    Takes a stream and presents a stream that is a slice of the original stream. 
 712    This is used when uploading media in chunks. In later versions of Python a 
 713    stream can be passed to httplib in place of the string of data to send. The 
 714    problem is that httplib just blindly reads to the end of the stream. This 
 715    wrapper presents a virtual stream that only reads to the end of the chunk. 
 716    """ 
 717   
 718 -  def __init__(self, stream, begin, chunksize): 
  719      """Constructor. 
 720   
 721      Args: 
 722        stream: (io.Base, file object), the stream to wrap. 
 723        begin: int, the seek position the chunk begins at. 
 724        chunksize: int, the size of the chunk. 
 725      """ 
 726      self._stream = stream 
 727      self._begin = begin 
 728      self._chunksize = chunksize 
 729      self._stream.seek(begin) 
  730   
 731 -  def read(self, n=-1): 
  732      """Read n bytes. 
 733   
 734      Args: 
 735        n, int, the number of bytes to read. 
 736   
 737      Returns: 
 738        A string of length 'n', or less if EOF is reached. 
 739      """ 
 740       
 741      cur = self._stream.tell() 
 742      end = self._begin + self._chunksize 
 743      if n == -1 or cur + n > end: 
 744        n = end - cur 
 745      return self._stream.read(n) 
   746   
 749    """Encapsulates a single HTTP request.""" 
 750   
 751    @util.positional(4) 
 752 -  def __init__(self, http, postproc, uri, 
 753                 method='GET', 
 754                 body=None, 
 755                 headers=None, 
 756                 methodId=None, 
 757                 resumable=None): 
  758      """Constructor for an HttpRequest. 
 759   
 760      Args: 
 761        http: httplib2.Http, the transport object to use to make a request 
 762        postproc: callable, called on the HTTP response and content to transform 
 763                  it into a data object before returning, or raising an exception 
 764                  on an error. 
 765        uri: string, the absolute URI to send the request to 
 766        method: string, the HTTP method to use 
 767        body: string, the request body of the HTTP request, 
 768        headers: dict, the HTTP request headers 
 769        methodId: string, a unique identifier for the API method being called. 
 770        resumable: MediaUpload, None if this is not a resumbale request. 
 771      """ 
 772      self.uri = uri 
 773      self.method = method 
 774      self.body = body 
 775      self.headers = headers or {} 
 776      self.methodId = methodId 
 777      self.http = http 
 778      self.postproc = postproc 
 779      self.resumable = resumable 
 780      self.response_callbacks = [] 
 781      self._in_error_state = False 
 782   
 783       
 784      self.body_size = len(self.body or '') 
 785   
 786       
 787      self.resumable_uri = None 
 788   
 789       
 790      self.resumable_progress = 0 
 791   
 792       
 793      self._rand = random.random 
 794      self._sleep = time.sleep 
  795   
 796    @util.positional(1) 
 797 -  def execute(self, http=None, num_retries=0): 
  798      """Execute the request. 
 799   
 800      Args: 
 801        http: httplib2.Http, an http object to be used in place of the 
 802              one the HttpRequest request object was constructed with. 
 803        num_retries: Integer, number of times to retry with randomized 
 804              exponential backoff. If all retries fail, the raised HttpError 
 805              represents the last request. If zero (default), we attempt the 
 806              request only once. 
 807   
 808      Returns: 
 809        A deserialized object model of the response body as determined 
 810        by the postproc. 
 811   
 812      Raises: 
 813        googleapiclient.errors.HttpError if the response was not a 2xx. 
 814        httplib2.HttpLib2Error if a transport error has occured. 
 815      """ 
 816      if http is None: 
 817        http = self.http 
 818   
 819      if self.resumable: 
 820        body = None 
 821        while body is None: 
 822          _, body = self.next_chunk(http=http, num_retries=num_retries) 
 823        return body 
 824   
 825       
 826   
 827      if 'content-length' not in self.headers: 
 828        self.headers['content-length'] = str(self.body_size) 
 829       
 830       
 831      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': 
 832        self.method = 'POST' 
 833        self.headers['x-http-method-override'] = 'GET' 
 834        self.headers['content-type'] = 'application/x-www-form-urlencoded' 
 835        parsed = urlparse(self.uri) 
 836        self.uri = urlunparse( 
 837            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, 
 838             None) 
 839            ) 
 840        self.body = parsed.query 
 841        self.headers['content-length'] = str(len(self.body)) 
 842   
 843       
 844      resp, content = _retry_request( 
 845            http, num_retries, 'request', self._sleep, self._rand, str(self.uri), 
 846            method=str(self.method), body=self.body, headers=self.headers) 
 847   
 848      for callback in self.response_callbacks: 
 849        callback(resp) 
 850      if resp.status >= 300: 
 851        raise HttpError(resp, content, uri=self.uri) 
 852      return self.postproc(resp, content) 
  853   
 854    @util.positional(2) 
 856      """add_response_headers_callback 
 857   
 858      Args: 
 859        cb: Callback to be called on receiving the response headers, of signature: 
 860   
 861        def cb(resp): 
 862          # Where resp is an instance of httplib2.Response 
 863      """ 
 864      self.response_callbacks.append(cb) 
  865   
 866    @util.positional(1) 
 868      """Execute the next step of a resumable upload. 
 869   
 870      Can only be used if the method being executed supports media uploads and 
 871      the MediaUpload object passed in was flagged as using resumable upload. 
 872   
 873      Example: 
 874   
 875        media = MediaFileUpload('cow.png', mimetype='image/png', 
 876                                chunksize=1000, resumable=True) 
 877        request = farm.animals().insert( 
 878            id='cow', 
 879            name='cow.png', 
 880            media_body=media) 
 881   
 882        response = None 
 883        while response is None: 
 884          status, response = request.next_chunk() 
 885          if status: 
 886            print "Upload %d%% complete." % int(status.progress() * 100) 
 887   
 888   
 889      Args: 
 890        http: httplib2.Http, an http object to be used in place of the 
 891              one the HttpRequest request object was constructed with. 
 892        num_retries: Integer, number of times to retry with randomized 
 893              exponential backoff. If all retries fail, the raised HttpError 
 894              represents the last request. If zero (default), we attempt the 
 895              request only once. 
 896   
 897      Returns: 
 898        (status, body): (ResumableMediaStatus, object) 
 899           The body will be None until the resumable media is fully uploaded. 
 900   
 901      Raises: 
 902        googleapiclient.errors.HttpError if the response was not a 2xx. 
 903        httplib2.HttpLib2Error if a transport error has occured. 
 904      """ 
 905      if http is None: 
 906        http = self.http 
 907   
 908      if self.resumable.size() is None: 
 909        size = '*' 
 910      else: 
 911        size = str(self.resumable.size()) 
 912   
 913      if self.resumable_uri is None: 
 914        start_headers = copy.copy(self.headers) 
 915        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 
 916        if size != '*': 
 917          start_headers['X-Upload-Content-Length'] = size 
 918        start_headers['content-length'] = str(self.body_size) 
 919   
 920        resp, content = _retry_request( 
 921            http, num_retries, 'resumable URI request', self._sleep, self._rand, 
 922            self.uri, method=self.method, body=self.body, headers=start_headers) 
 923   
 924        if resp.status == 200 and 'location' in resp: 
 925          self.resumable_uri = resp['location'] 
 926        else: 
 927          raise ResumableUploadError(resp, content) 
 928      elif self._in_error_state: 
 929         
 930         
 931         
 932        headers = { 
 933            'Content-Range': 'bytes */%s' % size, 
 934            'content-length': '0' 
 935            } 
 936        resp, content = http.request(self.resumable_uri, 'PUT', 
 937                                     headers=headers) 
 938        status, body = self._process_response(resp, content) 
 939        if body: 
 940           
 941          return (status, body) 
 942   
 943      if self.resumable.has_stream(): 
 944        data = self.resumable.stream() 
 945        if self.resumable.chunksize() == -1: 
 946          data.seek(self.resumable_progress) 
 947          chunk_end = self.resumable.size() - self.resumable_progress - 1 
 948        else: 
 949           
 950          data = _StreamSlice(data, self.resumable_progress, 
 951                              self.resumable.chunksize()) 
 952          chunk_end = min( 
 953              self.resumable_progress + self.resumable.chunksize() - 1, 
 954              self.resumable.size() - 1) 
 955      else: 
 956        data = self.resumable.getbytes( 
 957            self.resumable_progress, self.resumable.chunksize()) 
 958   
 959         
 960        if len(data) < self.resumable.chunksize(): 
 961          size = str(self.resumable_progress + len(data)) 
 962   
 963        chunk_end = self.resumable_progress + len(data) - 1 
 964   
 965      headers = { 
 966          'Content-Range': 'bytes %d-%d/%s' % ( 
 967              self.resumable_progress, chunk_end, size), 
 968           
 969           
 970          'Content-Length': str(chunk_end - self.resumable_progress + 1) 
 971          } 
 972   
 973      for retry_num in range(num_retries + 1): 
 974        if retry_num > 0: 
 975          self._sleep(self._rand() * 2**retry_num) 
 976          LOGGER.warning( 
 977              'Retry #%d for media upload: %s %s, following status: %d' 
 978              % (retry_num, self.method, self.uri, resp.status)) 
 979   
 980        try: 
 981          resp, content = http.request(self.resumable_uri, method='PUT', 
 982                                       body=data, 
 983                                       headers=headers) 
 984        except: 
 985          self._in_error_state = True 
 986          raise 
 987        if not _should_retry_response(resp.status, content): 
 988          break 
 989   
 990      return self._process_response(resp, content) 
  991   
 993      """Process the response from a single chunk upload. 
 994   
 995      Args: 
 996        resp: httplib2.Response, the response object. 
 997        content: string, the content of the response. 
 998   
 999      Returns: 
1000        (status, body): (ResumableMediaStatus, object) 
1001           The body will be None until the resumable media is fully uploaded. 
1002   
1003      Raises: 
1004        googleapiclient.errors.HttpError if the response was not a 2xx or a 308. 
1005      """ 
1006      if resp.status in [200, 201]: 
1007        self._in_error_state = False 
1008        return None, self.postproc(resp, content) 
1009      elif resp.status == 308: 
1010        self._in_error_state = False 
1011         
1012        try: 
1013          self.resumable_progress = int(resp['range'].split('-')[1]) + 1 
1014        except KeyError: 
1015           
1016          self.resumable_progress = 0 
1017        if 'location' in resp: 
1018          self.resumable_uri = resp['location'] 
1019      else: 
1020        self._in_error_state = True 
1021        raise HttpError(resp, content, uri=self.uri) 
1022   
1023      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 
1024              None) 
 1025   
1027      """Returns a JSON representation of the HttpRequest.""" 
1028      d = copy.copy(self.__dict__) 
1029      if d['resumable'] is not None: 
1030        d['resumable'] = self.resumable.to_json() 
1031      del d['http'] 
1032      del d['postproc'] 
1033      del d['_sleep'] 
1034      del d['_rand'] 
1035   
1036      return json.dumps(d) 
 1037   
1038    @staticmethod 
1040      """Returns an HttpRequest populated with info from a JSON object.""" 
1041      d = json.loads(s) 
1042      if d['resumable'] is not None: 
1043        d['resumable'] = MediaUpload.new_from_json(d['resumable']) 
1044      return HttpRequest( 
1045          http, 
1046          postproc, 
1047          uri=d['uri'], 
1048          method=d['method'], 
1049          body=d['body'], 
1050          headers=d['headers'], 
1051          methodId=d['methodId'], 
1052          resumable=d['resumable']) 
  1053   
1056    """Batches multiple HttpRequest objects into a single HTTP request. 
1057   
1058    Example: 
1059      from googleapiclient.http import BatchHttpRequest 
1060   
1061      def list_animals(request_id, response, exception): 
1062        \"\"\"Do something with the animals list response.\"\"\" 
1063        if exception is not None: 
1064          # Do something with the exception. 
1065          pass 
1066        else: 
1067          # Do something with the response. 
1068          pass 
1069   
1070      def list_farmers(request_id, response, exception): 
1071        \"\"\"Do something with the farmers list response.\"\"\" 
1072        if exception is not None: 
1073          # Do something with the exception. 
1074          pass 
1075        else: 
1076          # Do something with the response. 
1077          pass 
1078   
1079      service = build('farm', 'v2') 
1080   
1081      batch = BatchHttpRequest() 
1082   
1083      batch.add(service.animals().list(), list_animals) 
1084      batch.add(service.farmers().list(), list_farmers) 
1085      batch.execute(http=http) 
1086    """ 
1087   
1088    @util.positional(1) 
1089 -  def __init__(self, callback=None, batch_uri=None): 
 1090      """Constructor for a BatchHttpRequest. 
1091   
1092      Args: 
1093        callback: callable, A callback to be called for each response, of the 
1094          form callback(id, response, exception). The first parameter is the 
1095          request id, and the second is the deserialized response object. The 
1096          third is an googleapiclient.errors.HttpError exception object if an HTTP error 
1097          occurred while processing the request, or None if no error occurred. 
1098        batch_uri: string, URI to send batch requests to. 
1099      """ 
1100      if batch_uri is None: 
1101        batch_uri = _LEGACY_BATCH_URI 
1102   
1103      if batch_uri == _LEGACY_BATCH_URI: 
1104        LOGGER.warn( 
1105          "You have constructed a BatchHttpRequest using the legacy batch " 
1106          "endpoint %s. This endpoint will be turned down on March 25, 2019. " 
1107          "Please provide the API-specific endpoint or use " 
1108          "service.new_batch_http_request(). For more details see " 
1109          "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" 
1110          "and https://developers.google.com/api-client-library/python/guide/batch.", 
1111          _LEGACY_BATCH_URI) 
1112      self._batch_uri = batch_uri 
1113   
1114       
1115      self._callback = callback 
1116   
1117       
1118      self._requests = {} 
1119   
1120       
1121      self._callbacks = {} 
1122   
1123       
1124      self._order = [] 
1125   
1126       
1127      self._last_auto_id = 0 
1128   
1129       
1130      self._base_id = None 
1131   
1132       
1133      self._responses = {} 
1134   
1135       
1136      self._refreshed_credentials = {} 
 1137   
1139      """Refresh the credentials and apply to the request. 
1140   
1141      Args: 
1142        request: HttpRequest, the request. 
1143        http: httplib2.Http, the global http object for the batch. 
1144      """ 
1145       
1146       
1147       
1148      creds = None 
1149      request_credentials = False 
1150   
1151      if request.http is not None: 
1152        creds = _auth.get_credentials_from_http(request.http) 
1153        request_credentials = True 
1154   
1155      if creds is None and http is not None: 
1156        creds = _auth.get_credentials_from_http(http) 
1157   
1158      if creds is not None: 
1159        if id(creds) not in self._refreshed_credentials: 
1160          _auth.refresh_credentials(creds) 
1161          self._refreshed_credentials[id(creds)] = 1 
1162   
1163       
1164       
1165      if request.http is None or not request_credentials: 
1166        _auth.apply_credentials(creds, request.headers) 
 1167   
1168   
1170      """Convert an id to a Content-ID header value. 
1171   
1172      Args: 
1173        id_: string, identifier of individual request. 
1174   
1175      Returns: 
1176        A Content-ID header with the id_ encoded into it. A UUID is prepended to 
1177        the value because Content-ID headers are supposed to be universally 
1178        unique. 
1179      """ 
1180      if self._base_id is None: 
1181        self._base_id = uuid.uuid4() 
1182   
1183       
1184       
1185       
1186      return '<%s + %s>' % (self._base_id, quote(id_)) 
 1187   
1189      """Convert a Content-ID header value to an id. 
1190   
1191      Presumes the Content-ID header conforms to the format that _id_to_header() 
1192      returns. 
1193   
1194      Args: 
1195        header: string, Content-ID header value. 
1196   
1197      Returns: 
1198        The extracted id value. 
1199   
1200      Raises: 
1201        BatchError if the header is not in the expected format. 
1202      """ 
1203      if header[0] != '<' or header[-1] != '>': 
1204        raise BatchError("Invalid value for Content-ID: %s" % header) 
1205      if '+' not in header: 
1206        raise BatchError("Invalid value for Content-ID: %s" % header) 
1207      base, id_ = header[1:-1].split(' + ', 1) 
1208   
1209      return unquote(id_) 
 1210   
1212      """Convert an HttpRequest object into a string. 
1213   
1214      Args: 
1215        request: HttpRequest, the request to serialize. 
1216   
1217      Returns: 
1218        The request as a string in application/http format. 
1219      """ 
1220       
1221      parsed = urlparse(request.uri) 
1222      request_line = urlunparse( 
1223          ('', '', parsed.path, parsed.params, parsed.query, '') 
1224          ) 
1225      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 
1226      major, minor = request.headers.get('content-type', 'application/json').split('/') 
1227      msg = MIMENonMultipart(major, minor) 
1228      headers = request.headers.copy() 
1229   
1230      if request.http is not None: 
1231        credentials = _auth.get_credentials_from_http(request.http) 
1232        if credentials is not None: 
1233          _auth.apply_credentials(credentials, headers) 
1234   
1235       
1236      if 'content-type' in headers: 
1237        del headers['content-type'] 
1238   
1239      for key, value in six.iteritems(headers): 
1240        msg[key] = value 
1241      msg['Host'] = parsed.netloc 
1242      msg.set_unixfrom(None) 
1243   
1244      if request.body is not None: 
1245        msg.set_payload(request.body) 
1246        msg['content-length'] = str(len(request.body)) 
1247   
1248       
1249      fp = StringIO() 
1250       
1251      g = Generator(fp, maxheaderlen=0) 
1252      g.flatten(msg, unixfrom=False) 
1253      body = fp.getvalue() 
1254   
1255      return status_line + body 
 1256   
1258      """Convert string into httplib2 response and content. 
1259   
1260      Args: 
1261        payload: string, headers and body as a string. 
1262   
1263      Returns: 
1264        A pair (resp, content), such as would be returned from httplib2.request. 
1265      """ 
1266       
1267      status_line, payload = payload.split('\n', 1) 
1268      protocol, status, reason = status_line.split(' ', 2) 
1269   
1270       
1271      parser = FeedParser() 
1272      parser.feed(payload) 
1273      msg = parser.close() 
1274      msg['status'] = status 
1275   
1276       
1277      resp = httplib2.Response(msg) 
1278      resp.reason = reason 
1279      resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 
1280   
1281      content = payload.split('\r\n\r\n', 1)[1] 
1282   
1283      return resp, content 
 1284   
1286      """Create a new id. 
1287   
1288      Auto incrementing number that avoids conflicts with ids already used. 
1289   
1290      Returns: 
1291         string, a new unique id. 
1292      """ 
1293      self._last_auto_id += 1 
1294      while str(self._last_auto_id) in self._requests: 
1295        self._last_auto_id += 1 
1296      return str(self._last_auto_id) 
 1297   
1298    @util.positional(2) 
1299 -  def add(self, request, callback=None, request_id=None): 
 1300      """Add a new request. 
1301   
1302      Every callback added will be paired with a unique id, the request_id. That 
1303      unique id will be passed back to the callback when the response comes back 
1304      from the server. The default behavior is to have the library generate it's 
1305      own unique id. If the caller passes in a request_id then they must ensure 
1306      uniqueness for each request_id, and if they are not an exception is 
1307      raised. Callers should either supply all request_ids or never supply a 
1308      request id, to avoid such an error. 
1309   
1310      Args: 
1311        request: HttpRequest, Request to add to the batch. 
1312        callback: callable, A callback to be called for this response, of the 
1313          form callback(id, response, exception). The first parameter is the 
1314          request id, and the second is the deserialized response object. The 
1315          third is an googleapiclient.errors.HttpError exception object if an HTTP error 
1316          occurred while processing the request, or None if no errors occurred. 
1317        request_id: string, A unique id for the request. The id will be passed 
1318          to the callback with the response. 
1319   
1320      Returns: 
1321        None 
1322   
1323      Raises: 
1324        BatchError if a media request is added to a batch. 
1325        KeyError is the request_id is not unique. 
1326      """ 
1327   
1328      if len(self._order) >= MAX_BATCH_LIMIT: 
1329        raise BatchError("Exceeded the maximum calls(%d) in a single batch request." 
1330                         % MAX_BATCH_LIMIT) 
1331      if request_id is None: 
1332        request_id = self._new_id() 
1333      if request.resumable is not None: 
1334        raise BatchError("Media requests cannot be used in a batch request.") 
1335      if request_id in self._requests: 
1336        raise KeyError("A request with this ID already exists: %s" % request_id) 
1337      self._requests[request_id] = request 
1338      self._callbacks[request_id] = callback 
1339      self._order.append(request_id) 
 1340   
1341 -  def _execute(self, http, order, requests): 
 1342      """Serialize batch request, send to server, process response. 
1343   
1344      Args: 
1345        http: httplib2.Http, an http object to be used to make the request with. 
1346        order: list, list of request ids in the order they were added to the 
1347          batch. 
1348        request: list, list of request objects to send. 
1349   
1350      Raises: 
1351        httplib2.HttpLib2Error if a transport error has occured. 
1352        googleapiclient.errors.BatchError if the response is the wrong format. 
1353      """ 
1354      message = MIMEMultipart('mixed') 
1355       
1356      setattr(message, '_write_headers', lambda self: None) 
1357   
1358       
1359      for request_id in order: 
1360        request = requests[request_id] 
1361   
1362        msg = MIMENonMultipart('application', 'http') 
1363        msg['Content-Transfer-Encoding'] = 'binary' 
1364        msg['Content-ID'] = self._id_to_header(request_id) 
1365   
1366        body = self._serialize_request(request) 
1367        msg.set_payload(body) 
1368        message.attach(msg) 
1369   
1370       
1371       
1372      fp = StringIO() 
1373      g = Generator(fp, mangle_from_=False) 
1374      g.flatten(message, unixfrom=False) 
1375      body = fp.getvalue() 
1376   
1377      headers = {} 
1378      headers['content-type'] = ('multipart/mixed; ' 
1379                                 'boundary="%s"') % message.get_boundary() 
1380   
1381      resp, content = http.request(self._batch_uri, method='POST', body=body, 
1382                                   headers=headers) 
1383   
1384      if resp.status >= 300: 
1385        raise HttpError(resp, content, uri=self._batch_uri) 
1386   
1387       
1388      header = 'content-type: %s\r\n\r\n' % resp['content-type'] 
1389       
1390       
1391      if six.PY3: 
1392        content = content.decode('utf-8') 
1393      for_parser = header + content 
1394   
1395      parser = FeedParser() 
1396      parser.feed(for_parser) 
1397      mime_response = parser.close() 
1398   
1399      if not mime_response.is_multipart(): 
1400        raise BatchError("Response not in multipart/mixed format.", resp=resp, 
1401                         content=content) 
1402   
1403      for part in mime_response.get_payload(): 
1404        request_id = self._header_to_id(part['Content-ID']) 
1405        response, content = self._deserialize_response(part.get_payload()) 
1406         
1407        if isinstance(content, six.text_type): 
1408          content = content.encode('utf-8') 
1409        self._responses[request_id] = (response, content) 
 1410   
1411    @util.positional(1) 
1413      """Execute all the requests as a single batched HTTP request. 
1414   
1415      Args: 
1416        http: httplib2.Http, an http object to be used in place of the one the 
1417          HttpRequest request object was constructed with. If one isn't supplied 
1418          then use a http object from the requests in this batch. 
1419   
1420      Returns: 
1421        None 
1422   
1423      Raises: 
1424        httplib2.HttpLib2Error if a transport error has occured. 
1425        googleapiclient.errors.BatchError if the response is the wrong format. 
1426      """ 
1427       
1428      if len(self._order) == 0: 
1429        return None 
1430   
1431       
1432      if http is None: 
1433        for request_id in self._order: 
1434          request = self._requests[request_id] 
1435          if request is not None: 
1436            http = request.http 
1437            break 
1438   
1439      if http is None: 
1440        raise ValueError("Missing a valid http object.") 
1441   
1442       
1443       
1444      creds = _auth.get_credentials_from_http(http) 
1445      if creds is not None: 
1446        if not _auth.is_valid(creds): 
1447          LOGGER.info('Attempting refresh to obtain initial access_token') 
1448          _auth.refresh_credentials(creds) 
1449   
1450      self._execute(http, self._order, self._requests) 
1451   
1452       
1453       
1454      redo_requests = {} 
1455      redo_order = [] 
1456   
1457      for request_id in self._order: 
1458        resp, content = self._responses[request_id] 
1459        if resp['status'] == '401': 
1460          redo_order.append(request_id) 
1461          request = self._requests[request_id] 
1462          self._refresh_and_apply_credentials(request, http) 
1463          redo_requests[request_id] = request 
1464   
1465      if redo_requests: 
1466        self._execute(http, redo_order, redo_requests) 
1467   
1468       
1469       
1470       
1471   
1472      for request_id in self._order: 
1473        resp, content = self._responses[request_id] 
1474   
1475        request = self._requests[request_id] 
1476        callback = self._callbacks[request_id] 
1477   
1478        response = None 
1479        exception = None 
1480        try: 
1481          if resp.status >= 300: 
1482            raise HttpError(resp, content, uri=request.uri) 
1483          response = request.postproc(resp, content) 
1484        except HttpError as e: 
1485          exception = e 
1486   
1487        if callback is not None: 
1488          callback(request_id, response, exception) 
1489        if self._callback is not None: 
1490          self._callback(request_id, response, exception) 
  1491   
1494    """Mock of HttpRequest. 
1495   
1496    Do not construct directly, instead use RequestMockBuilder. 
1497    """ 
1498   
1499 -  def __init__(self, resp, content, postproc): 
 1500      """Constructor for HttpRequestMock 
1501   
1502      Args: 
1503        resp: httplib2.Response, the response to emulate coming from the request 
1504        content: string, the response body 
1505        postproc: callable, the post processing function usually supplied by 
1506                  the model class. See model.JsonModel.response() as an example. 
1507      """ 
1508      self.resp = resp 
1509      self.content = content 
1510      self.postproc = postproc 
1511      if resp is None: 
1512        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 
1513      if 'reason' in self.resp: 
1514        self.resp.reason = self.resp['reason'] 
 1515   
1517      """Execute the request. 
1518   
1519      Same behavior as HttpRequest.execute(), but the response is 
1520      mocked and not really from an HTTP request/response. 
1521      """ 
1522      return self.postproc(self.resp, self.content) 
  1523   
1526    """A simple mock of HttpRequest 
1527   
1528      Pass in a dictionary to the constructor that maps request methodIds to 
1529      tuples of (httplib2.Response, content, opt_expected_body) that should be 
1530      returned when that method is called. None may also be passed in for the 
1531      httplib2.Response, in which case a 200 OK response will be generated. 
1532      If an opt_expected_body (str or dict) is provided, it will be compared to 
1533      the body and UnexpectedBodyError will be raised on inequality. 
1534   
1535      Example: 
1536        response = '{"data": {"id": "tag:google.c...' 
1537        requestBuilder = RequestMockBuilder( 
1538          { 
1539            'plus.activities.get': (None, response), 
1540          } 
1541        ) 
1542        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 
1543   
1544      Methods that you do not supply a response for will return a 
1545      200 OK with an empty string as the response content or raise an excpetion 
1546      if check_unexpected is set to True. The methodId is taken from the rpcName 
1547      in the discovery document. 
1548   
1549      For more details see the project wiki. 
1550    """ 
1551   
1552 -  def __init__(self, responses, check_unexpected=False): 
 1553      """Constructor for RequestMockBuilder 
1554   
1555      The constructed object should be a callable object 
1556      that can replace the class HttpResponse. 
1557   
1558      responses - A dictionary that maps methodIds into tuples 
1559                  of (httplib2.Response, content). The methodId 
1560                  comes from the 'rpcName' field in the discovery 
1561                  document. 
1562      check_unexpected - A boolean setting whether or not UnexpectedMethodError 
1563                         should be raised on unsupplied method. 
1564      """ 
1565      self.responses = responses 
1566      self.check_unexpected = check_unexpected 
 1567   
1568 -  def __call__(self, http, postproc, uri, method='GET', body=None, 
1569                 headers=None, methodId=None, resumable=None): 
 1570      """Implements the callable interface that discovery.build() expects 
1571      of requestBuilder, which is to build an object compatible with 
1572      HttpRequest.execute(). See that method for the description of the 
1573      parameters and the expected response. 
1574      """ 
1575      if methodId in self.responses: 
1576        response = self.responses[methodId] 
1577        resp, content = response[:2] 
1578        if len(response) > 2: 
1579           
1580          expected_body = response[2] 
1581          if bool(expected_body) != bool(body): 
1582             
1583             
1584            raise UnexpectedBodyError(expected_body, body) 
1585          if isinstance(expected_body, str): 
1586            expected_body = json.loads(expected_body) 
1587          body = json.loads(body) 
1588          if body != expected_body: 
1589            raise UnexpectedBodyError(expected_body, body) 
1590        return HttpRequestMock(resp, content, postproc) 
1591      elif self.check_unexpected: 
1592        raise UnexpectedMethodError(methodId=methodId) 
1593      else: 
1594        model = JsonModel(False) 
1595        return HttpRequestMock(None, '{}', model.response) 
  1596   
1599    """Mock of httplib2.Http""" 
1600   
1601 -  def __init__(self, filename=None, headers=None): 
 1602      """ 
1603      Args: 
1604        filename: string, absolute filename to read response from 
1605        headers: dict, header to return with response 
1606      """ 
1607      if headers is None: 
1608        headers = {'status': '200'} 
1609      if filename: 
1610        f = open(filename, 'rb') 
1611        self.data = f.read() 
1612        f.close() 
1613      else: 
1614        self.data = None 
1615      self.response_headers = headers 
1616      self.headers = None 
1617      self.uri = None 
1618      self.method = None 
1619      self.body = None 
1620      self.headers = None 
 1621   
1622   
1623 -  def request(self, uri, 
1624                method='GET', 
1625                body=None, 
1626                headers=None, 
1627                redirections=1, 
1628                connection_type=None): 
 1629      self.uri = uri 
1630      self.method = method 
1631      self.body = body 
1632      self.headers = headers 
1633      return httplib2.Response(self.response_headers), self.data 
  1634   
1637    """Mock of httplib2.Http 
1638   
1639    Mocks a sequence of calls to request returning different responses for each 
1640    call. Create an instance initialized with the desired response headers 
1641    and content and then use as if an httplib2.Http instance. 
1642   
1643      http = HttpMockSequence([ 
1644        ({'status': '401'}, ''), 
1645        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 
1646        ({'status': '200'}, 'echo_request_headers'), 
1647        ]) 
1648      resp, content = http.request("http://examples.com") 
1649   
1650    There are special values you can pass in for content to trigger 
1651    behavours that are helpful in testing. 
1652   
1653    'echo_request_headers' means return the request headers in the response body 
1654    'echo_request_headers_as_json' means return the request headers in 
1655       the response body 
1656    'echo_request_body' means return the request body in the response body 
1657    'echo_request_uri' means return the request uri in the response body 
1658    """ 
1659   
1661      """ 
1662      Args: 
1663        iterable: iterable, a sequence of pairs of (headers, body) 
1664      """ 
1665      self._iterable = iterable 
1666      self.follow_redirects = True 
 1667   
1668 -  def request(self, uri, 
1669                method='GET', 
1670                body=None, 
1671                headers=None, 
1672                redirections=1, 
1673                connection_type=None): 
 1674      resp, content = self._iterable.pop(0) 
1675      if content == 'echo_request_headers': 
1676        content = headers 
1677      elif content == 'echo_request_headers_as_json': 
1678        content = json.dumps(headers) 
1679      elif content == 'echo_request_body': 
1680        if hasattr(body, 'read'): 
1681          content = body.read() 
1682        else: 
1683          content = body 
1684      elif content == 'echo_request_uri': 
1685        content = uri 
1686      if isinstance(content, six.text_type): 
1687        content = content.encode('utf-8') 
1688      return httplib2.Response(resp), content 
  1689   
1692    """Set the user-agent on every request. 
1693   
1694    Args: 
1695       http - An instance of httplib2.Http 
1696           or something that acts like it. 
1697       user_agent: string, the value for the user-agent header. 
1698   
1699    Returns: 
1700       A modified instance of http that was passed in. 
1701   
1702    Example: 
1703   
1704      h = httplib2.Http() 
1705      h = set_user_agent(h, "my-app-name/6.0") 
1706   
1707    Most of the time the user-agent will be set doing auth, this is for the rare 
1708    cases where you are accessing an unauthenticated endpoint. 
1709    """ 
1710    request_orig = http.request 
1711   
1712     
1713    def new_request(uri, method='GET', body=None, headers=None, 
1714                    redirections=httplib2.DEFAULT_MAX_REDIRECTS, 
1715                    connection_type=None): 
1716      """Modify the request headers to add the user-agent.""" 
1717      if headers is None: 
1718        headers = {} 
1719      if 'user-agent' in headers: 
1720        headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 
1721      else: 
1722        headers['user-agent'] = user_agent 
1723      resp, content = request_orig(uri, method=method, body=body, headers=headers, 
1724                          redirections=redirections, connection_type=connection_type) 
1725      return resp, content 
 1726   
1727    http.request = new_request 
1728    return http 
1729   
1732    """Tunnel PATCH requests over POST. 
1733    Args: 
1734       http - An instance of httplib2.Http 
1735           or something that acts like it. 
1736   
1737    Returns: 
1738       A modified instance of http that was passed in. 
1739   
1740    Example: 
1741   
1742      h = httplib2.Http() 
1743      h = tunnel_patch(h, "my-app-name/6.0") 
1744   
1745    Useful if you are running on a platform that doesn't support PATCH. 
1746    Apply this last if you are using OAuth 1.0, as changing the method 
1747    will result in a different signature. 
1748    """ 
1749    request_orig = http.request 
1750   
1751     
1752    def new_request(uri, method='GET', body=None, headers=None, 
1753                    redirections=httplib2.DEFAULT_MAX_REDIRECTS, 
1754                    connection_type=None): 
1755      """Modify the request headers to add the user-agent.""" 
1756      if headers is None: 
1757        headers = {} 
1758      if method == 'PATCH': 
1759        if 'oauth_token' in headers.get('authorization', ''): 
1760          LOGGER.warning( 
1761              'OAuth 1.0 request made with Credentials after tunnel_patch.') 
1762        headers['x-http-method-override'] = "PATCH" 
1763        method = 'POST' 
1764      resp, content = request_orig(uri, method=method, body=body, headers=headers, 
1765                          redirections=redirections, connection_type=connection_type) 
1766      return resp, content 
 1767   
1768    http.request = new_request 
1769    return http 
1770   
1773    """Builds httplib2.Http object 
1774   
1775    Returns: 
1776    A httplib2.Http object, which is used to make http requests, and which has timeout set by default. 
1777    To override default timeout call 
1778   
1779      socket.setdefaulttimeout(timeout_in_sec) 
1780   
1781    before interacting with this method. 
1782    """ 
1783    if socket.getdefaulttimeout() is not None: 
1784      http_timeout = socket.getdefaulttimeout() 
1785    else: 
1786      http_timeout = DEFAULT_HTTP_TIMEOUT_SEC 
1787    return httplib2.Http(timeout=http_timeout) 
 1788