This is an archive of the discontinued Mercurial Phabricator instance.

Differential D2220

convert: open all files in binary mode
ClosedPublic

Authored by durin42 on Feb 13 2018, 12:06 AM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHG42a393ea56d2: convert: open all files in binary mode

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

durin42 created this revision.Feb 13 2018, 12:06 AM
indygreg accepted this revision.Feb 13 2018, 1:01 AM
This revision is now accepted and ready to land.Feb 13 2018, 1:01 AM
Closed by commit rHG42a393ea56d2: convert: open all files in binary mode (authored by durin42). · Explain WhyFeb 13 2018, 1:03 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 5619

hgext/convert/common.py

self.fp = None self.fp = None
self.order = [] self.order = []
self._read() self._read()
def _read(self): def _read(self):
if not self.path: if not self.path:
return return
try: try:
fp = open(self.path, 'r') fp = open(self.path, 'rb')
except IOError as err: except IOError as err:
if err.errno != errno.ENOENT: if err.errno != errno.ENOENT:
raise raise
return return
for i, line in enumerate(util.iterfile(fp)): for i, line in enumerate(util.iterfile(fp)):
line = line.splitlines()[0].rstrip() line = line.splitlines()[0].rstrip()
if not line: if not line:
# Ignore blank lines # Ignore blank lines
continue continue
try: try:
key, value = line.rsplit(' ', 1) key, value = line.rsplit(' ', 1)
except ValueError: except ValueError:
raise error.Abort( raise error.Abort(
_('syntax error in %s(%d): key/value pair expected') _('syntax error in %s(%d): key/value pair expected')
% (self.path, i + 1)) % (self.path, i + 1))
if key not in self: if key not in self:
self.order.append(key) self.order.append(key)
super(mapfile, self).__setitem__(key, value) super(mapfile, self).__setitem__(key, value)
fp.close() fp.close()
def __setitem__(self, key, value): def __setitem__(self, key, value):
if self.fp is None: if self.fp is None:
try: try:
self.fp = open(self.path, 'a') self.fp = open(self.path, 'ab')
except IOError as err: except IOError as err:
raise error.Abort( raise error.Abort(
_('could not open map file %r: %s') % _('could not open map file %r: %s') %
(self.path, encoding.strtolocal(err.strerror))) (self.path, encoding.strtolocal(err.strerror)))
self.fp.write('%s %s\n' % (key, value)) self.fp.write('%s %s\n' % (key, value))
self.fp.flush() self.fp.flush()
super(mapfile, self).__setitem__(key, value) super(mapfile, self).__setitem__(key, value)

hgext/convert/convcmd.py

checkrevformat function in source files like checkrevformat function in source files like
hg.py, subversion.py etc. hg.py, subversion.py etc.
""" """
if not path: if not path:
return {} return {}
m = {} m = {}
try: try:
fp = open(path, 'r') fp = open(path, 'rb')
for i, line in enumerate(util.iterfile(fp)): for i, line in enumerate(util.iterfile(fp)):
line = line.splitlines()[0].rstrip() line = line.splitlines()[0].rstrip()
if not line: if not line:
# Ignore blank lines # Ignore blank lines
continue continue
# split line # split line
lex = shlex.shlex(line, posix=True) lex = shlex.shlex(line, posix=True)
lex.whitespace_split = True lex.whitespace_split = True
raise error.Abort(_("not all revisions were sorted")) raise error.Abort(_("not all revisions were sorted"))
return s return s
def writeauthormap(self): def writeauthormap(self):
authorfile = self.authorfile authorfile = self.authorfile
if authorfile: if authorfile:
self.ui.status(_('writing author map file %s\n') % authorfile) self.ui.status(_('writing author map file %s\n') % authorfile)
ofile = open(authorfile, 'w+') ofile = open(authorfile, 'wb+')
for author in self.authors: for author in self.authors:
ofile.write("%s=%s\n" % (author, self.authors[author])) ofile.write("%s=%s\n" % (author, self.authors[author]))
ofile.close() ofile.close()
def readauthormap(self, authorfile): def readauthormap(self, authorfile):
afile = open(authorfile, 'r') afile = open(authorfile, 'rb')
for line in afile: for line in afile:
line = line.strip() line = line.strip()
if not line or line.startswith('#'): if not line or line.startswith('#'):
continue continue
try: try:
srcauthor, dstauthor = line.split('=', 1) srcauthor, dstauthor = line.split('=', 1)

hgext/convert/cvs.py

checktool('cvs') checktool('cvs')
self.changeset = None self.changeset = None
self.files = {} self.files = {}
self.tags = {} self.tags = {}
self.lastbranch = {} self.lastbranch = {}
self.socket = None self.socket = None
self.cvsroot = open(os.path.join(cvs, "Root")).read()[:-1] self.cvsroot = open(os.path.join(cvs, "Root"), 'rb').read()[:-1]
self.cvsrepo = open(os.path.join(cvs, "Repository")).read()[:-1] self.cvsrepo = open(os.path.join(cvs, "Repository"), 'rb').read()[:-1]
self.encoding = encoding.encoding self.encoding = encoding.encoding
self._connect() self._connect()
def _parse(self): def _parse(self):
if self.changeset is not None: if self.changeset is not None:
return return
self.changeset = {} self.changeset = {}
port = int(port) port = int(port)
format0 = ":pserver:%s@%s:%s" % (user, serv, root) format0 = ":pserver:%s@%s:%s" % (user, serv, root)
format1 = ":pserver:%s@%s:%d%s" % (user, serv, port, root) format1 = ":pserver:%s@%s:%d%s" % (user, serv, port, root)
if not passw: if not passw:
passw = "A" passw = "A"
cvspass = os.path.expanduser("~/.cvspass") cvspass = os.path.expanduser("~/.cvspass")
try: try:
pf = open(cvspass) pf = open(cvspass, 'rb')
for line in pf.read().splitlines(): for line in pf.read().splitlines():
part1, part2 = line.split(' ', 1) part1, part2 = line.split(' ', 1)
# /1 :pserver:user@example.com:2401/cvsroot/foo # /1 :pserver:user@example.com:2401/cvsroot/foo
# Ah<Z # Ah<Z
if part1 == '/1': if part1 == '/1':
part1, part2 = part2.split(' ', 1) part1, part2 = part2.split(' ', 1)
format = format1 format = format1
# :pserver:user@example.com:/cvsroot/foo Ah<Z # :pserver:user@example.com:/cvsroot/foo Ah<Z

hgext/convert/cvsps.py

prefix = '' # leading path to strip of what we get from CVS prefix = '' # leading path to strip of what we get from CVS
if directory is None: if directory is None:
# Current working directory # Current working directory
# Get the real directory in the repository # Get the real directory in the repository
try: try:
prefix = open(os.path.join('CVS','Repository')).read().strip() prefix = open(os.path.join('CVS','Repository'), 'rb').read().strip()
directory = prefix directory = prefix
if prefix == ".": if prefix == ".":
prefix = "" prefix = ""
except IOError: except IOError:
raise logerror(_('not a CVS sandbox')) raise logerror(_('not a CVS sandbox'))
if prefix and not prefix.endswith(pycompat.ossep): if prefix and not prefix.endswith(pycompat.ossep):
prefix += pycompat.ossep prefix += pycompat.ossep
# Use the Root file in the sandbox, if it exists # Use the Root file in the sandbox, if it exists
try: try:
root = open(os.path.join('CVS','Root')).read().strip() root = open(os.path.join('CVS','Root'), 'rb').read().strip()
except IOError: except IOError:
pass pass
if not root: if not root:
root = encoding.environ.get('CVSROOT', '') root = encoding.environ.get('CVSROOT', '')
# read log cache if one exists # read log cache if one exists
oldlog = [] oldlog = []
cachefile = root.split(":") + [directory, "cache"] cachefile = root.split(":") + [directory, "cache"]
cachefile = ['-'.join(re.findall(r'\w+', s)) for s in cachefile if s] cachefile = ['-'.join(re.findall(r'\w+', s)) for s in cachefile if s]
cachefile = os.path.join(cachedir, cachefile = os.path.join(cachedir,
'.'.join([s for s in cachefile if s])) '.'.join([s for s in cachefile if s]))
if cache == 'update': if cache == 'update':
try: try:
ui.note(_('reading cvs log cache %s\n') % cachefile) ui.note(_('reading cvs log cache %s\n') % cachefile)
oldlog = pickle.load(open(cachefile)) oldlog = pickle.load(open(cachefile, 'rb'))
for e in oldlog: for e in oldlog:
if not (util.safehasattr(e, 'branchpoints') and if not (util.safehasattr(e, 'branchpoints') and
util.safehasattr(e, 'commitid') and util.safehasattr(e, 'commitid') and
util.safehasattr(e, 'mergepoint')): util.safehasattr(e, 'mergepoint')):
ui.status(_('ignoring old cache\n')) ui.status(_('ignoring old cache\n'))
oldlog = [] oldlog = []
break break
if oldlog and oldlog[-1].date >= log[0].date: if oldlog and oldlog[-1].date >= log[0].date:
raise logerror(_('log cache overlaps with new log entries,' raise logerror(_('log cache overlaps with new log entries,'
' re-run without cache.')) ' re-run without cache.'))
log = oldlog + log log = oldlog + log
# write the new cachefile # write the new cachefile
ui.note(_('writing cvs log cache %s\n') % cachefile) ui.note(_('writing cvs log cache %s\n') % cachefile)
pickle.dump(log, open(cachefile, 'w')) pickle.dump(log, open(cachefile, 'wb'))
else: else:
log = oldlog log = oldlog
ui.status(_('%d log entries\n') % len(log)) ui.status(_('%d log entries\n') % len(log))
encodings = ui.configlist('convert', 'cvsps.logencoding') encodings = ui.configlist('convert', 'cvsps.logencoding')
if encodings: if encodings:
def revstr(r): def revstr(r):

hgext/convert/filemap.py

return 1 return 1
if (name.startswith('/') or if (name.startswith('/') or
name.endswith('/') or name.endswith('/') or
'//' in name): '//' in name):
self.ui.warn(_('%s:%d: superfluous / in %s %r\n') % self.ui.warn(_('%s:%d: superfluous / in %s %r\n') %
(lex.infile, lex.lineno, listname, name)) (lex.infile, lex.lineno, listname, name))
return 1 return 1
return 0 return 0
lex = shlex.shlex(open(path), path, True) lex = shlex.shlex(open(path, 'rb'), path, True)
lex.wordchars += '!@#$%^&*()-=+[]{}|;:,./<>?' lex.wordchars += '!@#$%^&*()-=+[]{}|;:,./<>?'
cmd = lex.get_token() cmd = lex.get_token()
while cmd: while cmd:
if cmd == 'include': if cmd == 'include':
name = normalize(lex.get_token()) name = normalize(lex.get_token())
errs += check(name, self.exclude, 'exclude') errs += check(name, self.exclude, 'exclude')
self.include[name] = name self.include[name] = name
elif cmd == 'exclude': elif cmd == 'exclude':

hgext/convert/hg.py

if i == 0: if i == 0:
self._changescache = (rev, (ma, r)) self._changescache = (rev, (ma, r))
return ma + r return ma + r
def converted(self, rev, destrev): def converted(self, rev, destrev):
if self.convertfp is None: if self.convertfp is None:
self.convertfp = open(self.repo.vfs.join('shamap'), 'a') self.convertfp = open(self.repo.vfs.join('shamap'), 'ab')
self.convertfp.write('%s %s\n' % (destrev, rev)) self.convertfp.write('%s %s\n' % (destrev, rev))
self.convertfp.flush() self.convertfp.flush()
def before(self): def before(self):
self.ui.debug('run hg source pre-conversion action\n') self.ui.debug('run hg source pre-conversion action\n')
def after(self): def after(self):
self.ui.debug('run hg source post-conversion action\n') self.ui.debug('run hg source post-conversion action\n')

hgext/convert/subversion.py

return True return True
# Check to see if a given path is the root of an svn repo over http. We verify # Check to see if a given path is the root of an svn repo over http. We verify
# this by requesting a version-controlled URL we know can't exist and looking # this by requesting a version-controlled URL we know can't exist and looking
# for the svn-specific "not found" XML. # for the svn-specific "not found" XML.
def httpcheck(ui, path, proto): def httpcheck(ui, path, proto):
try: try:
opener = urlreq.buildopener() opener = urlreq.buildopener()
rsp = opener.open('%s://%s/!svn/ver/0/.svn' % (proto, path)) rsp = opener.open('%s://%s/!svn/ver/0/.svn' % (proto, path), 'rb')
data = rsp.read() data = rsp.read()
except urlerr.httperror as inst: except urlerr.httperror as inst:
if inst.code != 404: if inst.code != 404:
# Except for 404 we cannot know for sure this is not an svn repo # Except for 404 we cannot know for sure this is not an svn repo
ui.warn(_('svn: cannot probe remote repository, assume it could ' ui.warn(_('svn: cannot probe remote repository, assume it could '
'be a subversion repository. Use --source-type if you ' 'be a subversion repository. Use --source-type if you '
'know better.\n')) 'know better.\n'))
return True return True
stream.close() stream.close()
return tags return tags
def converted(self, rev, destrev): def converted(self, rev, destrev):
if not self.wc: if not self.wc:
return return
if self.convertfp is None: if self.convertfp is None:
self.convertfp = open(os.path.join(self.wc, '.svn', 'hg-shamap'), self.convertfp = open(os.path.join(self.wc, '.svn', 'hg-shamap'),
'a') 'ab')
self.convertfp.write('%s %d\n' % (destrev, self.revnum(rev))) self.convertfp.write('%s %d\n' % (destrev, self.revnum(rev)))
self.convertfp.flush() self.convertfp.flush()
def revid(self, revnum, module=None): def revid(self, revnum, module=None):
return 'svn:%s%s@%s' % (self.uuid, module or self.module, revnum) return 'svn:%s%s@%s' % (self.uuid, module or self.module, revnum)
def revnum(self, rev): def revnum(self, rev):
return int(rev.split('@')[-1]) return int(rev.split('@')[-1])
self.childmap = mapfile(ui, self.join('hg-childmap')) self.childmap = mapfile(ui, self.join('hg-childmap'))
if util.checkexec(self.wc): if util.checkexec(self.wc):
self.is_exec = util.isexec self.is_exec = util.isexec
else: else:
self.is_exec = None self.is_exec = None
if created: if created:
hook = os.path.join(created, 'hooks', 'pre-revprop-change') hook = os.path.join(created, 'hooks', 'pre-revprop-change')
fp = open(hook, 'w') fp = open(hook, 'wb')
fp.write(pre_revprop_change) fp.write(pre_revprop_change)
fp.close() fp.close()
util.setflags(hook, False, True) util.setflags(hook, False, True)
output = self.run0('info') output = self.run0('info')
self.uuid = self.uuid_re.search(output).group(1).strip() self.uuid = self.uuid_re.search(output).group(1).strip()
def wjoin(self, *names): def wjoin(self, *names):
if self.delexec: if self.delexec:
self.xargs(self.delexec, 'propdel', 'svn:executable') self.xargs(self.delexec, 'propdel', 'svn:executable')
self.delexec = [] self.delexec = []
if self.setexec: if self.setexec:
self.xargs(self.setexec, 'propset', 'svn:executable', '*') self.xargs(self.setexec, 'propset', 'svn:executable', '*')
self.setexec = [] self.setexec = []
fd, messagefile = tempfile.mkstemp(prefix='hg-convert-') fd, messagefile = tempfile.mkstemp(prefix='hg-convert-')
fp = os.fdopen(fd, pycompat.sysstr('w')) fp = os.fdopen(fd, pycompat.sysstr('wb'))
fp.write(commit.desc) fp.write(commit.desc)
fp.close() fp.close()
try: try:
output = self.run0('commit', output = self.run0('commit',
username=util.shortuser(commit.author), username=util.shortuser(commit.author),
file=messagefile, file=messagefile,
encoding='utf-8') encoding='utf-8')
try: try: