/atom/http.py

  1#!/usr/bin/python
  2#
  3# Copyright (C) 2008 Google Inc.
  4#
  5# Licensed under the Apache License, Version 2.0 (the "License");
  6# you may not use this file except in compliance with the License.
  7# You may obtain a copy of the License at
  8#
  9#      http://www.apache.org/licenses/LICENSE-2.0
 10#
 11# Unless required by applicable law or agreed to in writing, software
 12# distributed under the License is distributed on an "AS IS" BASIS,
 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14# See the License for the specific language governing permissions and
 15# limitations under the License.
 16
 17
 18"""HttpClients in this module use httplib to make HTTP requests.
 19
 20This module make HTTP requests based on httplib, but there are environments
 21in which an httplib based approach will not work (if running in Google App
 22Engine for example). In those cases, higher level classes (like AtomService
 23and GDataService) can swap out the HttpClient to transparently use a 
 24different mechanism for making HTTP requests.
 25
 26  HttpClient: Contains a request method which performs an HTTP call to the 
 27      server.
 28      
 29  ProxiedHttpClient: Contains a request method which connects to a proxy using
 30      settings stored in operating system environment variables then 
 31      performs an HTTP call to the endpoint server.
 32"""
 33
 34
 35__author__ = 'api.jscudder (Jeff Scudder)'
 36
 37
 38import types
 39import os
 40import httplib
 41import atom.url
 42import atom.http_interface
 43import socket
 44import base64
 45import atom.http_core
 46ssl_imported = False
 47ssl = None
 48try:
 49  import ssl
 50  ssl_imported = True
 51except ImportError:
 52  pass
 53  
 54
 55
 56class ProxyError(atom.http_interface.Error):
 57  pass
 58
 59
 60class TestConfigurationError(Exception):
 61  pass
 62
 63
 64DEFAULT_CONTENT_TYPE = 'application/atom+xml'
 65
 66
 67class HttpClient(atom.http_interface.GenericHttpClient):
 68  # Added to allow old v1 HttpClient objects to use the new 
 69  # http_code.HttpClient. Used in unit tests to inject a mock client.
 70  v2_http_client = None
 71
 72  def __init__(self, headers=None):
 73    self.debug = False
 74    self.headers = headers or {}
 75
 76  def request(self, operation, url, data=None, headers=None):
 77    """Performs an HTTP call to the server, supports GET, POST, PUT, and 
 78    DELETE.
 79
 80    Usage example, perform and HTTP GET on http://www.google.com/:
 81      import atom.http
 82      client = atom.http.HttpClient()
 83      http_response = client.request('GET', 'http://www.google.com/')
 84
 85    Args:
 86      operation: str The HTTP operation to be performed. This is usually one
 87          of 'GET', 'POST', 'PUT', or 'DELETE'
 88      data: filestream, list of parts, or other object which can be converted
 89          to a string. Should be set to None when performing a GET or DELETE.
 90          If data is a file-like object which can be read, this method will 
 91          read a chunk of 100K bytes at a time and send them. 
 92          If the data is a list of parts to be sent, each part will be 
 93          evaluated and sent.
 94      url: The full URL to which the request should be sent. Can be a string
 95          or atom.url.Url.
 96      headers: dict of strings. HTTP headers which should be sent
 97          in the request. 
 98    """
 99    all_headers = self.headers.copy()
100    if headers:
101      all_headers.update(headers)
102
103    # If the list of headers does not include a Content-Length, attempt to
104    # calculate it based on the data object.
105    if data and 'Content-Length' not in all_headers:
106      if isinstance(data, types.StringTypes):
107        all_headers['Content-Length'] = str(len(data))
108      else:
109        raise atom.http_interface.ContentLengthRequired('Unable to calculate '
110            'the length of the data parameter. Specify a value for '
111            'Content-Length')
112
113    # Set the content type to the default value if none was set.
114    if 'Content-Type' not in all_headers:
115      all_headers['Content-Type'] = DEFAULT_CONTENT_TYPE
116
117    if self.v2_http_client is not None:
118      http_request = atom.http_core.HttpRequest(method=operation)
119      atom.http_core.Uri.parse_uri(str(url)).modify_request(http_request)
120      http_request.headers = all_headers
121      if data:
122        http_request._body_parts.append(data)
123      return self.v2_http_client.request(http_request=http_request)
124
125    if not isinstance(url, atom.url.Url):
126      if isinstance(url, types.StringTypes):
127        url = atom.url.parse_url(url)
128      else:
129        raise atom.http_interface.UnparsableUrlObject('Unable to parse url '
130            'parameter because it was not a string or atom.url.Url')
131    
132    connection = self._prepare_connection(url, all_headers)
133
134    if self.debug:
135      connection.debuglevel = 1
136
137    connection.putrequest(operation, self._get_access_url(url), 
138        skip_host=True)
139    if url.port is not None:
140      connection.putheader('Host', '%s:%s' % (url.host, url.port))
141    else:
142      connection.putheader('Host', url.host)
143
144    # Overcome a bug in Python 2.4 and 2.5
145    # httplib.HTTPConnection.putrequest adding
146    # HTTP request header 'Host: www.google.com:443' instead of
147    # 'Host: www.google.com', and thus resulting the error message
148    # 'Token invalid - AuthSub token has wrong scope' in the HTTP response.
149    if (url.protocol == 'https' and int(url.port or 443) == 443 and
150        hasattr(connection, '_buffer') and
151        isinstance(connection._buffer, list)):
152      header_line = 'Host: %s:443' % url.host
153      replacement_header_line = 'Host: %s' % url.host
154      try:
155        connection._buffer[connection._buffer.index(header_line)] = (
156            replacement_header_line)
157      except ValueError:  # header_line missing from connection._buffer
158        pass
159
160    # Send the HTTP headers.
161    for header_name in all_headers:
162      connection.putheader(header_name, all_headers[header_name])
163    connection.endheaders()
164
165    # If there is data, send it in the request.
166    if data:
167      if isinstance(data, list):
168        for data_part in data:
169          _send_data_part(data_part, connection)
170      else:
171        _send_data_part(data, connection)
172
173    # Return the HTTP Response from the server.
174    return connection.getresponse()
175    
176  def _prepare_connection(self, url, headers):
177    if not isinstance(url, atom.url.Url):
178      if isinstance(url, types.StringTypes):
179        url = atom.url.parse_url(url)
180      else:
181        raise atom.http_interface.UnparsableUrlObject('Unable to parse url '
182            'parameter because it was not a string or atom.url.Url')
183    if url.protocol == 'https':
184      if not url.port:
185        return httplib.HTTPSConnection(url.host)
186      return httplib.HTTPSConnection(url.host, int(url.port))
187    else:
188      if not url.port:
189        return httplib.HTTPConnection(url.host)
190      return httplib.HTTPConnection(url.host, int(url.port))
191
192  def _get_access_url(self, url):
193    return url.to_string()
194
195
196class ProxiedHttpClient(HttpClient):
197  """Performs an HTTP request through a proxy.
198  
199  The proxy settings are obtained from enviroment variables. The URL of the 
200  proxy server is assumed to be stored in the environment variables 
201  'https_proxy' and 'http_proxy' respectively. If the proxy server requires
202  a Basic Auth authorization header, the username and password are expected to 
203  be in the 'proxy-username' or 'proxy_username' variable and the 
204  'proxy-password' or 'proxy_password' variable.
205  
206  After connecting to the proxy server, the request is completed as in 
207  HttpClient.request.
208  """
209  def _prepare_connection(self, url, headers):
210    proxy_auth = _get_proxy_auth()
211    if url.protocol == 'https':
212      # destination is https
213      proxy = os.environ.get('https_proxy')
214      if proxy:
215        # Set any proxy auth headers 
216        if proxy_auth:
217          proxy_auth = 'Proxy-authorization: %s' % proxy_auth
218          
219        # Construct the proxy connect command.
220        port = url.port
221        if not port:
222          port = '443'
223        proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (url.host, port)
224        
225        # Set the user agent to send to the proxy
226        if headers and 'User-Agent' in headers:
227          user_agent = 'User-Agent: %s\r\n' % (headers['User-Agent'])
228        else:
229          user_agent = ''
230        
231        proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent)
232        
233        # Find the proxy host and port.
234        proxy_url = atom.url.parse_url(proxy)
235        if not proxy_url.port:
236          proxy_url.port = '80'
237        
238        # Connect to the proxy server, very simple recv and error checking
239        p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
240        p_sock.connect((proxy_url.host, int(proxy_url.port)))
241        p_sock.sendall(proxy_pieces)
242        response = ''
243
244        # Wait for the full response.
245        while response.find("\r\n\r\n") == -1:
246          response += p_sock.recv(8192)
247       
248        p_status = response.split()[1]
249        if p_status != str(200):
250          raise ProxyError('Error status=%s' % str(p_status))
251
252        # Trivial setup for ssl socket.
253        sslobj = None
254        if ssl_imported:
255          sslobj = ssl.wrap_socket(p_sock, None, None)
256        else:
257          sock_ssl = socket.ssl(p_sock, None, None)
258          sslobj = httplib.FakeSocket(p_sock, sock_ssl)
259 
260        # Initalize httplib and replace with the proxy socket.
261        connection = httplib.HTTPConnection(proxy_url.host)
262        connection.sock = sslobj
263        return connection
264      else:
265        # The request was HTTPS, but there was no https_proxy set.
266        return HttpClient._prepare_connection(self, url, headers)
267    else:
268      proxy = os.environ.get('http_proxy')
269      if proxy:
270        # Find the proxy host and port.
271        proxy_url = atom.url.parse_url(proxy)
272        if not proxy_url.port:
273          proxy_url.port = '80'
274        
275        if proxy_auth:
276          headers['Proxy-Authorization'] = proxy_auth.strip()
277
278        return httplib.HTTPConnection(proxy_url.host, int(proxy_url.port))
279      else:
280        # The request was HTTP, but there was no http_proxy set.
281        return HttpClient._prepare_connection(self, url, headers)
282
283  def _get_access_url(self, url):
284    return url.to_string()
285
286
287def _get_proxy_auth():
288  proxy_username = os.environ.get('proxy-username')
289  if not proxy_username:
290    proxy_username = os.environ.get('proxy_username')
291  proxy_password = os.environ.get('proxy-password')
292  if not proxy_password:
293    proxy_password = os.environ.get('proxy_password')
294  if proxy_username:
295    user_auth = base64.encodestring('%s:%s' % (proxy_username,
296                                               proxy_password))
297    return 'Basic %s\r\n' % (user_auth.strip())
298  else:
299    return ''
300
301
302def _send_data_part(data, connection):
303  if isinstance(data, types.StringTypes):
304    connection.send(data)
305    return
306  # Check to see if data is a file-like object that has a read method.
307  elif hasattr(data, 'read'):
308    # Read the file and send it a chunk at a time.
309    while 1:
310      binarydata = data.read(100000)
311      if binarydata == '': break
312      connection.send(binarydata)
313    return
314  else:
315    # The data object was not a file.
316    # Try to convert to a string and send the data.
317    connection.send(str(data))
318    return