Remove httplib2, replace with Requests (#3674)

* Core: remove httplib2, replace with Requests

Additionally remove make_exception in favor of from_http_status and from_http_response.

* Datastore: replace httplib2 with Requests

* DNS: replace httplib2 with Requests

* Error Reporting: replace httplib2 with requests

* Language: replace httplib2 with Requests

* Logging: replace httplib2 with requests

* Monitoring: replace httplib2 with Requests

* Pubsub: replace httplib2 with Requests

* Resource Manager: replace httplib2 with Requests

* Runtimeconfig: replace httplib2 with Requests

* Speech: replace httplib2 with Requests

* Storage: replace httplib2 with Requests

* BigQuery: replace httplib2 with Requests

* Translate: replace httplib2 with Requests

* Vision: replace httplib2 with Requests

Author: Jon Wayne Parrott

packages/google-cloud-core/google/cloud/_helpers.py

@@ -25,12 +25,11 @@
 import re
 from threading import local as Local
 
-import google_auth_httplib2
-import httplib2
 import six
 from six.moves import http_client
 
 import google.auth
+import google.auth.transport.requests
 from google.protobuf import duration_pb2
 from google.protobuf import timestamp_pb2
 
@@ -550,7 +549,7 @@ def make_secure_channel(credentials, user_agent, host, extra_options=()):
     :returns: gRPC secure channel with credentials attached.
     """
     target = '%s:%d' % (host, http_client.HTTPS_PORT)
-    http_request = google_auth_httplib2.Request(http=httplib2.Http())
+    http_request = google.auth.transport.requests.Request()
 
     user_agent_option = ('grpc.primary_user_agent', user_agent)
     options = (user_agent_option,) + extra_options

packages/google-cloud-core/google/cloud/_http.py

@@ -18,10 +18,9 @@
 import platform
 
 from pkg_resources import get_distribution
-import six
 from six.moves.urllib.parse import urlencode
 
-from google.cloud.exceptions import make_exception
+from google.cloud import exceptions
 
 
 API_BASE_URL = 'https://www.googleapis.com'
@@ -67,8 +66,9 @@ def credentials(self):
     def http(self):
         """A getter for the HTTP transport used in talking to the API.
 
-        :rtype: :class:`httplib2.Http`
-        :returns: A Http object used to transport data.
+        Returns:
+            google.auth.transport.requests.AuthorizedSession:
+                A :class:`requests.Session` instance.
         """
         return self._client._http
 
@@ -168,23 +168,13 @@ def _make_request(self, method, url, data=None, content_type=None,
             custom behavior, for example, to defer an HTTP request and complete
             initialization of the object at a later time.
 
-        :rtype: tuple of ``response`` (a dictionary of sorts)
-                and ``content`` (a string).
-        :returns: The HTTP response object and the content of the response,
-                  returned by :meth:`_do_request`.
+        :rtype: :class:`requests.Response`
+        :returns: The HTTP response.
         """
         headers = headers or {}
         headers.update(self._EXTRA_HEADERS)
         headers['Accept-Encoding'] = 'gzip'
 
-        if data:
-            content_length = len(str(data))
-        else:
-            content_length = 0
-
-        # NOTE: str is intended, bytes are sufficient for headers.
-        headers['Content-Length'] = str(content_length)
-
         if content_type:
             headers['Content-Type'] = content_type
 
@@ -215,12 +205,11 @@ def _do_request(self, method, url, headers, data,
             (Optional) Unused ``target_object`` here but may be used by a
             superclass.
 
-        :rtype: tuple of ``response`` (a dictionary of sorts)
-                and ``content`` (a string).
-        :returns: The HTTP response object and the content of the response.
+        :rtype: :class:`requests.Response`
+        :returns: The HTTP response.
         """
-        return self.http.request(uri=url, method=method, headers=headers,
-                                 body=data)
+        return self.http.request(
+            url=url, method=method, headers=headers, data=data)
 
     def api_request(self, method, path, query_params=None,
                     data=None, content_type=None, headers=None,
@@ -281,7 +270,7 @@ def api_request(self, method, path, query_params=None,
 
         :raises ~google.cloud.exceptions.GoogleCloudError: if the response code
             is not 200 OK.
-        :raises TypeError: if the response content type is not JSON.
+        :raises ValueError: if the response content type is not JSON.
         :rtype: dict or str
         :returns: The API response payload, either as a raw string or
                   a dictionary if the response is valid JSON.
@@ -296,21 +285,14 @@ def api_request(self, method, path, query_params=None,
             data = json.dumps(data)
             content_type = 'application/json'
 
-        response, content = self._make_request(
+        response = self._make_request(
             method=method, url=url, data=data, content_type=content_type,
             headers=headers, target_object=_target_object)
 
-        if not 200 <= response.status < 300:
-            raise make_exception(response, content,
-                                 error_info=method + ' ' + url)
+        if not 200 <= response.status_code < 300:
+            raise exceptions.from_http_response(response)
 
-        string_or_bytes = (six.binary_type, six.text_type)
-        if content and expect_json and isinstance(content, string_or_bytes):
-            content_type = response.get('content-type', '')
-            if not content_type.startswith('application/json'):
-                raise TypeError('Expected JSON, got %s' % content_type)
-            if isinstance(content, six.binary_type):
-                content = content.decode('utf-8')
-            return json.loads(content)
-
-        return content
+        if expect_json and response.content:
+            return response.json()
+        else:
+            return response.content

packages/google-cloud-core/google/cloud/client.py

@@ -18,11 +18,11 @@
 import json
 from pickle import PicklingError
 
-import google_auth_httplib2
 import six
 
 import google.auth
 import google.auth.credentials
+import google.auth.transport.requests
 from google.cloud._helpers import _determine_default_project
 from google.oauth2 import service_account
 
@@ -87,36 +87,23 @@ class Client(_ClientFactoryMixin):
     Stores ``credentials`` and an HTTP object so that subclasses
     can pass them along to a connection class.
 
-    If no value is passed in for ``_http``, a :class:`httplib2.Http` object
+    If no value is passed in for ``_http``, a :class:`requests.Session` object
     will be created and authorized with the ``credentials``. If not, the
     ``credentials`` and ``_http`` need not be related.
 
     Callers and subclasses may seek to use the private key from
     ``credentials`` to sign data.
 
-    A custom (non-``httplib2``) HTTP object must have a ``request`` method
-    which accepts the following arguments:
-
-    * ``uri``
-    * ``method``
-    * ``body``
-    * ``headers``
-
-    In addition, ``redirections`` and ``connection_type`` may be used.
-
-    A custom ``_http`` object will also need to be able to add a bearer token
-    to API requests and handle token refresh on 401 errors.
-
     :type credentials: :class:`~google.auth.credentials.Credentials`
     :param credentials: (Optional) The OAuth2 Credentials to use for this
                         client. If not passed (and if no ``_http`` object is
                         passed), falls back to the default inferred from the
                         environment.
 
-    :type _http: :class:`~httplib2.Http`
+    :type _http: :class:`~requests.Session`
     :param _http: (Optional) HTTP object to make requests. Can be any object
                   that defines ``request()`` with the same interface as
-                  :meth:`~httplib2.Http.request`. If not passed, an
+                  :meth:`requests.Session.request`. If not passed, an
                   ``_http`` object is created that is bound to the
                   ``credentials`` for the current object.
                   This parameter should be considered private, and could
@@ -151,12 +138,13 @@ def __getstate__(self):
     def _http(self):
         """Getter for object used for HTTP transport.
 
-        :rtype: :class:`~httplib2.Http`
+        :rtype: :class:`~requests.Session`
         :returns: An HTTP object.
         """
         if self._http_internal is None:
-            self._http_internal = google_auth_httplib2.AuthorizedHttp(
-                self._credentials)
+            self._http_internal = (
+                google.auth.transport.requests.AuthorizedSession(
+                    self._credentials))
         return self._http_internal
 
 
@@ -204,10 +192,10 @@ class ClientWithProject(Client, _ClientProjectMixin):
                         passed), falls back to the default inferred from the
                         environment.
 
-    :type _http: :class:`~httplib2.Http`
+    :type _http: :class:`~requests.Session`
     :param _http: (Optional) HTTP object to make requests. Can be any object
                   that defines ``request()`` with the same interface as
-                  :meth:`~httplib2.Http.request`. If not passed, an
+                  :meth:`~requests.Session.request`. If not passed, an
                   ``_http`` object is created that is bound to the
                   ``credentials`` for the current object.
                   This parameter should be considered private, and could

packages/google-cloud-core/google/cloud/exceptions.py

@@ -21,7 +21,6 @@
 from __future__ import absolute_import
 
 import copy
-import json
 
 import six
 
@@ -186,56 +185,55 @@ class GatewayTimeout(ServerError):
     code = 504
 
 
-def make_exception(response, content, error_info=None, use_json=True):
-    """Factory:  create exception based on HTTP response code.
+def from_http_status(status_code, message, errors=()):
+    """Create a :class:`GoogleCloudError` from an HTTP status code.
 
-    :type response: :class:`httplib2.Response` or other HTTP response object
-    :param response: A response object that defines a status code as the
-                     status attribute.
+    Args:
+        status_code (int): The HTTP status code.
+        message (str): The exception message.
+        errors (Sequence[Any]): A list of additional error information.
+
+    Returns:
+        GoogleCloudError: An instance of the appropriate subclass of
+            :class:`GoogleCloudError`.
+    """
+    error_class = _HTTP_CODE_TO_EXCEPTION.get(status_code, GoogleCloudError)
+    error = error_class(message, errors)
+
+    if error.code is None:
+        error.code = status_code
+
+    return error
 
-    :type content: str or dictionary
-    :param content: The body of the HTTP error response.
 
-    :type error_info: str
-    :param error_info: Optional string giving extra information about the
-                       failed request.
+def from_http_response(response):
+    """Create a :class:`GoogleCloudError` from a :class:`requests.Response`.
 
-    :type use_json: bool
-    :param use_json: Flag indicating if ``content`` is expected to be JSON.
+    Args:
+        response (requests.Response): The HTTP response.
 
-    :rtype: instance of :class:`GoogleCloudError`, or a concrete subclass.
-    :returns: Exception specific to the error response.
+    Returns:
+        GoogleCloudError: An instance of the appropriate subclass of
+            :class:`GoogleCloudError`, with the message and errors populated
+            from the response.
     """
-    if isinstance(content, six.binary_type):
-        content = content.decode('utf-8')
-
-    if isinstance(content, six.string_types):
-        payload = None
-        if use_json:
-            try:
-                payload = json.loads(content)
-            except ValueError:
-                # Expected JSON but received something else.
-                pass
-        if payload is None:
-            payload = {'error': {'message': content}}
-    else:
-        payload = content
-
-    message = payload.get('error', {}).get('message', '')
+    try:
+        payload = response.json()
+    except ValueError:
+        payload = {'error': {'message': response.text or 'unknown error'}}
+
+    error_message = payload.get('error', {}).get('message', 'unknown error')
     errors = payload.get('error', {}).get('errors', ())
 
-    if error_info is not None:
-        message += ' (%s)' % (error_info,)
+    message = '{method} {url}: {error}'.format(
+        method=response.request.method,
+        url=response.request.url,
+        error=error_message)
 
-    try:
-        klass = _HTTP_CODE_TO_EXCEPTION[response.status]
-    except KeyError:
-        error = GoogleCloudError(message, errors)
-        error.code = response.status
-    else:
-        error = klass(message, errors)
-    return error
+    exception = from_http_status(
+        response.status_code, message, errors=errors)
+    exception.response = response
+    return exception
 
 
 def _walk_subclasses(klass):

packages/google-cloud-core/setup.py

@@ -51,11 +51,10 @@
 
 
 REQUIREMENTS = [
-    'httplib2 >= 0.9.1',
     'googleapis-common-protos >= 1.3.4',
     'protobuf >= 3.0.0',
     'google-auth >= 0.4.0, < 2.0.0dev',
-    'google-auth-httplib2',
+    'requests >= 2.4.0, < 3.0.0dev',
     'six',
     'tenacity >= 4.0.0, <5.0.0dev'
 ]