This is an archive of the discontinued Mercurial Phabricator instance.

Differential D891

cleanup: use urllibcompat for renamed methods on urllib request objects
ClosedPublic

Authored by durin42 on Oct 1 2017, 12:43 PM.

Details

Reviewers
yuja
krbullock
Group Reviewers
hg-reviewers
Commits
rHG1232f7fa00c3: cleanup: use urllibcompat for renamed methods on urllib request objects
Summary

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durin42 created this revision.Oct 1 2017, 12:43 PM
yuja accepted this revision.Oct 2 2017, 12:01 PM
yuja added a subscriber: yuja.

Maybe we'll have to work around unicode/bytes dance in urllibcompat.py, but
this patch looks good as itself.

This revision is now accepted and ready to land.Oct 2 2017, 12:01 PM
durin42 updated this revision to Diff 2424.Oct 4 2017, 12:26 PM
krbullock accepted this revision.Oct 4 2017, 3:48 PM
Closed by commit rHG1232f7fa00c3: cleanup: use urllibcompat for renamed methods on urllib request objects (authored by durin42). · Explain WhyOct 4 2017, 3:49 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/byterange.py (7 lines)
Mmercurial/httpconnection.py (18 lines)
Mmercurial/keepalive.py (15 lines)
Mmercurial/url.py (16 lines)

Diff 2430

mercurial/byterange.py

import ftplib import ftplib
import mimetypes import mimetypes
import os import os
import re import re
import socket import socket
import stat import stat
from . import ( from . import (
urllibcompat,
util, util,
) )
urlerr = util.urlerr urlerr = util.urlerr
urlreq = util.urlreq urlreq = util.urlreq
addclosehook = urlreq.addclosehook addclosehook = urlreq.addclosehook
addinfourl = urlreq.addinfourl addinfourl = urlreq.addinfourl
pos += bufsize pos += bufsize
class FileRangeHandler(urlreq.filehandler): class FileRangeHandler(urlreq.filehandler):
"""FileHandler subclass that adds Range support. """FileHandler subclass that adds Range support.
This class handles Range headers exactly like an HTTP This class handles Range headers exactly like an HTTP
server would. server would.
""" """
def open_local_file(self, req): def open_local_file(self, req):
host = req.get_host() host = urllibcompat.gethost(req)
file = req.get_selector() file = urllibcompat.getselector(req)
localfile = urlreq.url2pathname(file) localfile = urlreq.url2pathname(file)
stats = os.stat(localfile) stats = os.stat(localfile)
size = stats[stat.ST_SIZE] size = stats[stat.ST_SIZE]
modified = email.Utils.formatdate(stats[stat.ST_MTIME]) modified = email.Utils.formatdate(stats[stat.ST_MTIME])
mtype = mimetypes.guess_type(file)[0] mtype = mimetypes.guess_type(file)[0]
if host: if host:
host, port = urlreq.splitport(host) host, port = urlreq.splitport(host)
if port or socket.gethostbyname(host) not in self.get_names(): if port or socket.gethostbyname(host) not in self.get_names():
# Unfortunately, a large amount of base FTP code had to be copied # Unfortunately, a large amount of base FTP code had to be copied
# from urllib and urllib2 in order to insert the FTP REST command. # from urllib and urllib2 in order to insert the FTP REST command.
# Code modifications for range support have been commented as # Code modifications for range support have been commented as
# follows: # follows:
# -- range support modifications start/end here # -- range support modifications start/end here
class FTPRangeHandler(urlreq.ftphandler): class FTPRangeHandler(urlreq.ftphandler):
def ftp_open(self, req): def ftp_open(self, req):
host = req.get_host() host = urllibcompat.gethost(req)
if not host: if not host:
raise IOError('ftp error', 'no host given') raise IOError('ftp error', 'no host given')
host, port = splitport(host) host, port = splitport(host)
if port is None: if port is None:
port = ftplib.FTP_PORT port = ftplib.FTP_PORT
else: else:
port = int(port) port = int(port)

mercurial/httpconnection.py

import logging import logging
import os import os
import socket import socket
from .i18n import _ from .i18n import _
from . import ( from . import (
httpclient, httpclient,
sslutil, sslutil,
urllibcompat,
util, util,
) )
urlerr = util.urlerr urlerr = util.urlerr
urlreq = util.urlreq urlreq = util.urlreq
# moved here from url.py to avoid a cycle # moved here from url.py to avoid a cycle
class httpsendfile(object): class httpsendfile(object):
- code: HTTP status code - code: HTTP status code
""" """
# If using a proxy, the host returned by get_host() is # If using a proxy, the host returned by get_host() is
# actually the proxy. On Python 2.6.1, the real destination # actually the proxy. On Python 2.6.1, the real destination
# hostname is encoded in the URI in the urllib2 request # hostname is encoded in the URI in the urllib2 request
# object. On Python 2.6.5, it's stored in the _tunnel_host # object. On Python 2.6.5, it's stored in the _tunnel_host
# attribute which has no accessor. # attribute which has no accessor.
tunhost = getattr(req, '_tunnel_host', None) tunhost = getattr(req, '_tunnel_host', None)
host = req.get_host() host = urllibcompat.gethost(req)
if tunhost: if tunhost:
proxyhost = host proxyhost = host
host = tunhost host = tunhost
elif req.has_proxy(): elif req.has_proxy():
proxyhost = req.get_host() proxyhost = urllibcompat.gethost(req)
host = req.get_selector().split('://', 1)[1].split('/', 1)[0] host = urllibcompat.getselector(
req).split('://', 1)[1].split('/', 1)[0]
else: else:
proxyhost = None proxyhost = None
if proxyhost: if proxyhost:
if ':' in proxyhost: if ':' in proxyhost:
# Note: this means we'll explode if we try and use an # Note: this means we'll explode if we try and use an
# IPv6 http proxy. This isn't a regression, so we # IPv6 http proxy. This isn't a regression, so we
# won't worry about it for now. # won't worry about it for now.
h = http_class(host, timeout=timeout, proxy_hostport=proxy) h = http_class(host, timeout=timeout, proxy_hostport=proxy)
self._connections.setdefault(connkey, []).append(h) self._connections.setdefault(connkey, []).append(h)
headers = dict(req.headers) headers = dict(req.headers)
headers.update(req.unredirected_hdrs) headers.update(req.unredirected_hdrs)
headers = dict( headers = dict(
(name.title(), val) for name, val in headers.items()) (name.title(), val) for name, val in headers.items())
try: try:
path = req.get_selector() path = urllibcompat.getselector(req)
if '://' in path: if '://' in path:
path = path.split('://', 1)[1].split('/', 1)[1] path = path.split('://', 1)[1].split('/', 1)[1]
if path[0] != '/': if path[0] != '/':
path = '/' + path path = '/' + path
h.request(req.get_method(), path, req.data, headers) h.request(req.get_method(), path, req.data, headers)
r = h.getresponse() r = h.getresponse()
except socket.error as err: # XXX what error? except socket.error as err: # XXX what error?
raise urlerr.urlerror(err) raise urlerr.urlerror(err)
# Pick apart the HTTPResponse object to get the addinfourl # Pick apart the HTTPResponse object to get the addinfourl
# object initialized properly. # object initialized properly.
r.recv = r.read r.recv = r.read
resp = urlreq.addinfourl(r, r.headers, req.get_full_url()) resp = urlreq.addinfourl(r, r.headers, urllibcompat.getfullurl(req))
resp.code = r.status resp.code = r.status
resp.msg = r.reason resp.msg = r.reason
return resp return resp
# httplib always uses the given host/port as the socket connect # httplib always uses the given host/port as the socket connect
# target, and then allows full URIs in the request path, which it # target, and then allows full URIs in the request path, which it
# then observes and treats as a signal to do proxying instead. # then observes and treats as a signal to do proxying instead.
def http_open(self, req): def http_open(self, req):
if req.get_full_url().startswith('https'): if urllibcompat.getfullurl(req).startswith('https'):
return self.https_open(req) return self.https_open(req)
def makehttpcon(*args, **kwargs): def makehttpcon(*args, **kwargs):
k2 = dict(kwargs) k2 = dict(kwargs)
k2['use_ssl'] = False k2['use_ssl'] = False
return HTTPConnection(*args, **k2) return HTTPConnection(*args, **k2)
return self.do_open(makehttpcon, req, False) return self.do_open(makehttpcon, req, False)
def https_open(self, req): def https_open(self, req):
# req.get_full_url() does not contain credentials and we may # urllibcompat.getfullurl(req) does not contain credentials and we may
# need them to match the certificates. # need them to match the certificates.
url = req.get_full_url() url = urllibcompat.getfullurl(req)
user, password = self.pwmgr.find_stored_password(url) user, password = self.pwmgr.find_stored_password(url)
res = readauthforuri(self.ui, url, user) res = readauthforuri(self.ui, url, user)
if res: if res:
group, auth = res group, auth = res
self.auth = auth self.auth = auth
self.ui.debug("using auth.%s.* for authentication\n" % group) self.ui.debug("using auth.%s.* for authentication\n" % group)
else: else:
self.auth = None self.auth = None

mercurial/keepalive.py

import hashlib import hashlib
import socket import socket
import sys import sys
import threading import threading
from .i18n import _ from .i18n import _
from . import ( from . import (
pycompat, pycompat,
urllibcompat,
util, util,
) )
httplib = util.httplib httplib = util.httplib
urlerr = util.urlerr urlerr = util.urlerr
urlreq = util.urlreq urlreq = util.urlreq
DEBUG = None DEBUG = None
connection.close() connection.close()
self._cm.remove(connection) self._cm.remove(connection)
#### Transaction Execution #### Transaction Execution
def http_open(self, req): def http_open(self, req):
return self.do_open(HTTPConnection, req) return self.do_open(HTTPConnection, req)
def do_open(self, http_class, req): def do_open(self, http_class, req):
host = req.get_host() host = urllibcompat.gethost(req)
if not host: if not host:
raise urlerr.urlerror('no host given') raise urlerr.urlerror('no host given')
try: try:
h = self._cm.get_ready_conn(host) h = self._cm.get_ready_conn(host)
while h: while h:
r = self._reuse_connection(h, req, host) r = self._reuse_connection(h, req, host)
headers.update(sorted(req.headers.items())) headers.update(sorted(req.headers.items()))
headers.update(sorted(req.unredirected_hdrs.items())) headers.update(sorted(req.unredirected_hdrs.items()))
headers = util.sortdict((n.lower(), v) for n, v in headers.items()) headers = util.sortdict((n.lower(), v) for n, v in headers.items())
skipheaders = {} skipheaders = {}
for n in ('host', 'accept-encoding'): for n in ('host', 'accept-encoding'):
if n in headers: if n in headers:
skipheaders['skip_' + n.replace('-', '_')] = 1 skipheaders['skip_' + n.replace('-', '_')] = 1
try: try:
if req.has_data(): if urllibcompat.hasdata(req):
data = req.get_data() data = urllibcompat.getdata(req)
h.putrequest( h.putrequest(
req.get_method(), req.get_selector(), **skipheaders) req.get_method(), urllibcompat.getselector(req),
**skipheaders)
if 'content-type' not in headers: if 'content-type' not in headers:
h.putheader('Content-type', h.putheader('Content-type',
'application/x-www-form-urlencoded') 'application/x-www-form-urlencoded')
if 'content-length' not in headers: if 'content-length' not in headers:
h.putheader('Content-length', '%d' % len(data)) h.putheader('Content-length', '%d' % len(data))
else: else:
h.putrequest( h.putrequest(
req.get_method(), req.get_selector(), **skipheaders) req.get_method(), urllibcompat.getselector(req),
**skipheaders)
except socket.error as err: except socket.error as err:
raise urlerr.urlerror(err) raise urlerr.urlerror(err)
for k, v in headers.items(): for k, v in headers.items():
h.putheader(k, v) h.putheader(k, v)
h.endheaders() h.endheaders()
if req.has_data(): if urllibcompat.hasdata(req):
h.send(data) h.send(data)
class HTTPHandler(KeepAliveHandler, urlreq.httphandler): class HTTPHandler(KeepAliveHandler, urlreq.httphandler):
pass pass
class HTTPResponse(httplib.HTTPResponse): class HTTPResponse(httplib.HTTPResponse):
# we need to subclass HTTPResponse in order to # we need to subclass HTTPResponse in order to
# 1) add readline() and readlines() methods # 1) add readline() and readlines() methods

mercurial/url.py

from .i18n import _ from .i18n import _
from . import ( from . import (
encoding, encoding,
error, error,
httpconnection as httpconnectionmod, httpconnection as httpconnectionmod,
keepalive, keepalive,
pycompat, pycompat,
sslutil, sslutil,
urllibcompat,
util, util,
) )
httplib = util.httplib httplib = util.httplib
stringio = util.stringio stringio = util.stringio
urlerr = util.urlerr urlerr = util.urlerr
urlreq = util.urlreq urlreq = util.urlreq
(proxy.host, proxy.port)) (proxy.host, proxy.port))
else: else:
proxies = {} proxies = {}
urlreq.proxyhandler.__init__(self, proxies) urlreq.proxyhandler.__init__(self, proxies)
self.ui = ui self.ui = ui
def proxy_open(self, req, proxy, type_): def proxy_open(self, req, proxy, type_):
host = req.get_host().split(':')[0] host = urllibcompat.gethost(req).split(':')[0]
for e in self.no_list: for e in self.no_list:
if host == e: if host == e:
return None return None
if e.startswith('*.') and host.endswith(e[2:]): if e.startswith('*.') and host.endswith(e[2:]):
return None return None
if e.startswith('.') and host.endswith(e[1:]): if e.startswith('.') and host.endswith(e[1:]):
return None return None
# HTTPS proxying before and after Python 2.6.3. # HTTPS proxying before and after Python 2.6.3.
def _generic_start_transaction(handler, h, req): def _generic_start_transaction(handler, h, req):
tunnel_host = getattr(req, '_tunnel_host', None) tunnel_host = getattr(req, '_tunnel_host', None)
if tunnel_host: if tunnel_host:
if tunnel_host[:7] not in ['http://', 'https:/']: if tunnel_host[:7] not in ['http://', 'https:/']:
tunnel_host = 'https://' + tunnel_host tunnel_host = 'https://' + tunnel_host
new_tunnel = True new_tunnel = True
else: else:
tunnel_host = req.get_selector() tunnel_host = urllibcompat.getselector(req)
new_tunnel = False new_tunnel = False
if new_tunnel or tunnel_host == req.get_full_url(): # has proxy if new_tunnel or tunnel_host == urllibcompat.getfullurl(req): # has proxy
u = util.url(tunnel_host) u = util.url(tunnel_host)
if new_tunnel or u.scheme == 'https': # only use CONNECT for HTTPS if new_tunnel or u.scheme == 'https': # only use CONNECT for HTTPS
h.realhostport = ':'.join([u.host, (u.port or '443')]) h.realhostport = ':'.join([u.host, (u.port or '443')])
h.headers = req.headers.copy() h.headers = req.headers.copy()
h.headers.update(handler.parent.addheaders) h.headers.update(handler.parent.addheaders)
return return
h.realhostport = None h.realhostport = None
self.pwmgr = passwordmgr(self.ui, self.pwmgr = passwordmgr(self.ui,
self.ui.httppasswordmgrdb) self.ui.httppasswordmgrdb)
def _start_transaction(self, h, req): def _start_transaction(self, h, req):
_generic_start_transaction(self, h, req) _generic_start_transaction(self, h, req)
return keepalive.KeepAliveHandler._start_transaction(self, h, req) return keepalive.KeepAliveHandler._start_transaction(self, h, req)
def https_open(self, req): def https_open(self, req):
# req.get_full_url() does not contain credentials and we may # urllibcompat.getfullurl() does not contain credentials
# need them to match the certificates. # and we may need them to match the certificates.
url = req.get_full_url() url = urllibcompat.getfullurl(req)
user, password = self.pwmgr.find_stored_password(url) user, password = self.pwmgr.find_stored_password(url)
res = httpconnectionmod.readauthforuri(self.ui, url, user) res = httpconnectionmod.readauthforuri(self.ui, url, user)
if res: if res:
group, auth = res group, auth = res
self.auth = auth self.auth = auth
self.ui.debug("using auth.%s.* for authentication\n" % group) self.ui.debug("using auth.%s.* for authentication\n" % group)
else: else:
self.auth = None self.auth = None
# Reset the retry counter once for each request. # Reset the retry counter once for each request.
if req is not self.retried_req: if req is not self.retried_req:
self.retried_req = req self.retried_req = req
self.retried = 0 self.retried = 0
return urlreq.httpbasicauthhandler.http_error_auth_reqed( return urlreq.httpbasicauthhandler.http_error_auth_reqed(
self, auth_header, host, req, headers) self, auth_header, host, req, headers)
def retry_http_basic_auth(self, host, req, realm): def retry_http_basic_auth(self, host, req, realm):
user, pw = self.passwd.find_user_password(realm, req.get_full_url()) user, pw = self.passwd.find_user_password(
realm, urllibcompat.getfullurl(req))
if pw is not None: if pw is not None:
raw = "%s:%s" % (user, pw) raw = "%s:%s" % (user, pw)
auth = 'Basic %s' % base64.b64encode(raw).strip() auth = 'Basic %s' % base64.b64encode(raw).strip()
if req.get_header(self.auth_header, None) == auth: if req.get_header(self.auth_header, None) == auth:
return None return None
self.auth = auth self.auth = auth
req.add_unredirected_header(self.auth_header, auth) req.add_unredirected_header(self.auth_header, auth)
return self.parent.open(req) return self.parent.open(req)