( )⚙ D12249 py3: use pickle directly

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D12249

py3: use pickle directly
ClosedPublic

Authored by indygreg on Mar 1 2022, 7:35 PM.

Details

Reviewers
durin42
Alphare
Group Reviewers
hg-reviewers
Commits
rHGdf56e6bd37f6: py3: use pickle directly
Summary

pycompat.pickle abstracted over the different pickle modules in
Python 2 and 3. Now that we're Python 3 only, it is safe to use the
pickle module directly. So this commit does that.

As part of this we remove the rules from check-code.py that were
forbidden direct pickle module use.

We retain the util.pickle symbol for backwards compatibility, just
in case some extensions were using it.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

indygreg created this revision.Mar 1 2022, 7:35 PM
indygreg updated this revision to Diff 32335.Mar 1 2022, 11:29 PM
Alphare accepted this revision.Mar 2 2022, 5:45 AM
This revision is now accepted and ready to land.Mar 2 2022, 5:45 AM
indygreg added a commit: rHGdf56e6bd37f6: py3: use pickle directly.Mar 3 2022, 11:44 AM
Closed by commit rHGdf56e6bd37f6: py3: use pickle directly (authored by indygreg). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 32324

contrib/check-code.py

), ),
(r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"), (r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"),
( (
r'^import SocketServer', r'^import SocketServer',
"don't use SockerServer, use util.socketserver", "don't use SockerServer, use util.socketserver",
), ),
(r'^import urlparse', "don't use urlparse, use util.urlreq"), (r'^import urlparse', "don't use urlparse, use util.urlreq"),
(r'^import xmlrpclib', "don't use xmlrpclib, use util.xmlrpclib"), (r'^import xmlrpclib', "don't use xmlrpclib, use util.xmlrpclib"),
(r'^import cPickle', "don't use cPickle, use util.pickle"),
(r'^import pickle', "don't use pickle, use util.pickle"),
(r'^import httplib', "don't use httplib, use util.httplib"), (r'^import httplib', "don't use httplib, use util.httplib"),
(r'^import BaseHTTPServer', "use util.httpserver instead"), (r'^import BaseHTTPServer', "use util.httpserver instead"),
( (
r'^(from|import) mercurial\.(cext|pure|cffi)', r'^(from|import) mercurial\.(cext|pure|cffi)',
"use mercurial.policy.importmod instead", "use mercurial.policy.importmod instead",
), ),
(r'\.next\(\)', "don't use .next(), use next(...)"), (r'\.next\(\)', "don't use .next(), use next(...)"),
( (

hgext/convert/common.py

# common.py - common code for the convert extension # common.py - common code for the convert extension
# #
# Copyright 2005-2009 Olivia Mackall <olivia@selenic.com> and others # Copyright 2005-2009 Olivia Mackall <olivia@selenic.com> and others
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import base64 import base64
import datetime import datetime
import errno import errno
import os import os
import pickle
import re import re
import shlex import shlex
import subprocess import subprocess
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
encoding, encoding,
error, error,
phases, phases,
pycompat, pycompat,
util, util,
) )
from mercurial.utils import procutil from mercurial.utils import procutil
pickle = util.pickle
propertycache = util.propertycache propertycache = util.propertycache
def _encodeornone(d): def _encodeornone(d):
if d is None: if d is None:
return return
return d.encode('latin1') return d.encode('latin1')

hgext/convert/cvsps.py

# Mercurial built-in replacement for cvsps. # Mercurial built-in replacement for cvsps.
# #
# Copyright 2008, Frank Kingswood <frank@kingswood-consulting.co.uk> # Copyright 2008, Frank Kingswood <frank@kingswood-consulting.co.uk>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import functools import functools
import os import os
import pickle
import re import re
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
encoding, encoding,
error, error,
hook, hook,
pycompat, pycompat,
util, util,
) )
from mercurial.utils import ( from mercurial.utils import (
dateutil, dateutil,
procutil, procutil,
stringutil, stringutil,
) )
pickle = util.pickle
class logentry(object): class logentry(object):
"""Class logentry has the following attributes: """Class logentry has the following attributes:
.author - author name as CVS knows it .author - author name as CVS knows it
.branch - name of branch this revision is on .branch - name of branch this revision is on
.branches - revision tuple of branches starting at this revision .branches - revision tuple of branches starting at this revision
.comment - commit message .comment - commit message
.commitid - CVS commitid or None .commitid - CVS commitid or None

hgext/convert/subversion.py

# Subversion 1.4/1.5 Python API backend # Subversion 1.4/1.5 Python API backend
# #
# Copyright(C) 2007 Daniel Holth et al # Copyright(C) 2007 Daniel Holth et al
from __future__ import absolute_import from __future__ import absolute_import
import codecs import codecs
import locale import locale
import os import os
import pickle
import re import re
import xml.dom.minidom import xml.dom.minidom
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
encoding, encoding,
error, error,
pycompat, pycompat,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from mercurial.utils import ( from mercurial.utils import (
dateutil, dateutil,
procutil, procutil,
stringutil, stringutil,
) )
from . import common from . import common
pickle = util.pickle
stringio = util.stringio stringio = util.stringio
propertycache = util.propertycache propertycache = util.propertycache
urlerr = util.urlerr urlerr = util.urlerr
urlreq = util.urlreq urlreq = util.urlreq
commandline = common.commandline commandline = common.commandline
commit = common.commit commit = common.commit
converter_sink = common.converter_sink converter_sink = common.converter_sink

hgext/histedit.py

import fcntl import fcntl
import termios import termios
except ImportError: except ImportError:
fcntl = None fcntl = None
termios = None termios = None
import functools import functools
import os import os
import pickle
import struct import struct
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import ( from mercurial.pycompat import (
getattr, getattr,
open, open,
) )
from mercurial.node import ( from mercurial.node import (
util, util,
) )
from mercurial.utils import ( from mercurial.utils import (
dateutil, dateutil,
stringutil, stringutil,
urlutil, urlutil,
) )
pickle = util.pickle
cmdtable = {} cmdtable = {}
command = registrar.command(cmdtable) command = registrar.command(cmdtable)
configtable = {} configtable = {}
configitem = registrar.configitem(configtable) configitem = registrar.configitem(configtable)
configitem( configitem(
b'experimental', b'experimental',
b'histedit.autoverb', b'histedit.autoverb',

mercurial/formatter.py

baz: foo, bar baz: foo, bar
""" """
from __future__ import absolute_import, print_function from __future__ import absolute_import, print_function
import contextlib import contextlib
import itertools import itertools
import os import os
import pickle
from .i18n import _ from .i18n import _
from .node import ( from .node import (
hex, hex,
short, short,
) )
from .thirdparty import attr from .thirdparty import attr
from . import ( from . import (
error, error,
pycompat, pycompat,
templatefilters, templatefilters,
templatekw, templatekw,
templater, templater,
templateutil, templateutil,
util, util,
) )
from .utils import ( from .utils import (
cborutil, cborutil,
dateutil, dateutil,
stringutil, stringutil,
) )
pickle = util.pickle
def isprintable(obj): def isprintable(obj):
"""Check if the given object can be directly passed in to formatter's """Check if the given object can be directly passed in to formatter's
write() and data() functions write() and data() functions
Returns False if the object is unsupported or must be pre-processed by Returns False if the object is unsupported or must be pre-processed by
formatdate(), formatdict(), or formatlist(). formatdate(), formatdict(), or formatlist().
""" """

mercurial/util.py

import contextlib import contextlib
import errno import errno
import gc import gc
import hashlib import hashlib
import itertools import itertools
import locale import locale
import mmap import mmap
import os import os
import pickle
import platform as pyplatform import platform as pyplatform
import re as remod import re as remod
import shutil import shutil
import stat import stat
import sys import sys
import time import time
import traceback import traceback
import warnings import warnings
base85 = policy.importmod('base85') base85 = policy.importmod('base85')
osutil = policy.importmod('osutil') osutil = policy.importmod('osutil')
b85decode = base85.b85decode b85decode = base85.b85decode
b85encode = base85.b85encode b85encode = base85.b85encode
cookielib = pycompat.cookielib cookielib = pycompat.cookielib
httplib = pycompat.httplib httplib = pycompat.httplib
pickle = pycompat.pickle
safehasattr = pycompat.safehasattr safehasattr = pycompat.safehasattr
socketserver = pycompat.socketserver socketserver = pycompat.socketserver
bytesio = pycompat.bytesio bytesio = pycompat.bytesio
# TODO deprecate stringio name, as it is a lie on Python 3. # TODO deprecate stringio name, as it is a lie on Python 3.
stringio = bytesio stringio = bytesio
xmlrpclib = pycompat.xmlrpclib xmlrpclib = pycompat.xmlrpclib
httpserver = urllibcompat.httpserver httpserver = urllibcompat.httpserver

mercurial/worker.py

# worker.py - master-slave parallelism support # worker.py - master-slave parallelism support
# #
# Copyright 2013 Facebook, Inc. # Copyright 2013 Facebook, Inc.
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import errno import errno
import os import os
import pickle
import signal import signal
import sys import sys
import threading import threading
import time import time
try: try:
import selectors import selectors
signal.signal(signal.SIGCHLD, oldchldhandler) signal.signal(signal.SIGCHLD, oldchldhandler)
def workerfunc(): def workerfunc():
for r, w in pipes[:-1]: for r, w in pipes[:-1]:
os.close(r) os.close(r)
os.close(w) os.close(w)
os.close(rfd) os.close(rfd)
for result in func(*(staticargs + (pargs,))): for result in func(*(staticargs + (pargs,))):
os.write(wfd, util.pickle.dumps(result)) os.write(wfd, pickle.dumps(result))
return 0 return 0
ret = scmutil.callcatch(ui, workerfunc) ret = scmutil.callcatch(ui, workerfunc)
except: # parent re-raises, child never returns except: # parent re-raises, child never returns
if os.getpid() == parentpid: if os.getpid() == parentpid:
raise raise
exctype = sys.exc_info()[0] exctype = sys.exc_info()[0]
force = not issubclass(exctype, KeyboardInterrupt) force = not issubclass(exctype, KeyboardInterrupt)
selector.close() selector.close()
return problem[0] return problem[0]
try: try:
openpipes = len(pipes) openpipes = len(pipes)
while openpipes > 0: while openpipes > 0:
for key, events in selector.select(): for key, events in selector.select():
try: try:
res = util.pickle.load(_blockingreader(key.fileobj)) res = pickle.load(_blockingreader(key.fileobj))
if hasretval and res[0]: if hasretval and res[0]:
retval.update(res[1]) retval.update(res[1])
else: else:
yield res yield res
except EOFError: except EOFError:
selector.unregister(key.fileobj) selector.unregister(key.fileobj)
key.fileobj.close() key.fileobj.close()
openpipes -= 1 openpipes -= 1

tests/test-status.t

"itemtype": "file", "itemtype": "file",
"path": "modified", "path": "modified",
"status": "C" "status": "C"
} }
] ]
$ hg status -A -Tpickle > pickle $ hg status -A -Tpickle > pickle
>>> from __future__ import print_function >>> from __future__ import print_function
>>> import pickle
>>> from mercurial import util >>> from mercurial import util
>>> pickle = util.pickle
>>> data = sorted((x[b'status'].decode(), x[b'path'].decode()) for x in pickle.load(open("pickle", r"rb"))) >>> data = sorted((x[b'status'].decode(), x[b'path'].decode()) for x in pickle.load(open("pickle", r"rb")))
>>> for s, p in data: print("%s %s" % (s, p)) >>> for s, p in data: print("%s %s" % (s, p))
! deleted ! deleted
? pickle ? pickle
? unknown ? unknown
A added A added
A copied A copied
C .hgignore C .hgignore

tests/test-stdio.py

#!/usr/bin/env python #!/usr/bin/env python
""" """
Tests the buffering behavior of stdio streams in `mercurial.utils.procutil`. Tests the buffering behavior of stdio streams in `mercurial.utils.procutil`.
""" """
from __future__ import absolute_import from __future__ import absolute_import
import contextlib import contextlib
import errno import errno
import os import os
import pickle
import signal import signal
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import unittest import unittest
from mercurial import pycompat, util from mercurial import pycompat, util
assert stream in ('stdout', 'stderr') assert stream in ('stdout', 'stderr')
def check_output(stream_receiver, proc): def check_output(stream_receiver, proc):
os.close(stream_receiver) os.close(stream_receiver)
proc.stdin.write(b'x') proc.stdin.write(b'x')
proc.stdin.close() proc.stdin.close()
def post_child_check(): def post_child_check():
err = util.pickle.load(err_f) err = pickle.load(err_f)
self.assertEqual(err.errno, errno.EPIPE) self.assertEqual(err.errno, errno.EPIPE)
self.assertEqual(err.strerror, "Broken pipe") self.assertEqual(err.strerror, "Broken pipe")
with tempfile.NamedTemporaryFile('rb') as err_f: with tempfile.NamedTemporaryFile('rb') as err_f:
self._test( self._test(
TEST_BROKEN_PIPE_CHILD_SCRIPT.format( TEST_BROKEN_PIPE_CHILD_SCRIPT.format(
stream=stream, err_fn=err_f.name stream=stream, err_fn=err_f.name
), ),