This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7201

fsmonitor: refresh pywatchman with upstream
ClosedPublic

Authored by indygreg on Nov 2 2019, 6:03 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG6469c23a40a2: fsmonitor: refresh pywatchman with upstream
Summary

This commit vendors pywatchman commit 259dc66dc9591f9b7ce76d0275bb1065f390c9b1
from upstream without modifications. The previously vendored pywatchman
from changeset 16f4b341288d was from Git commit c77452.

This commit effectively undoes the following Mercurial changesets:

  • dd35abc409ee fsmonitor: correct an error message
  • b1f62cd39b5c fsmonitor: layer on another hack in bser.c for os.stat() compat (issue5811)
  • c31ce080eb75 py3: convert arguments, cwd and env to native strings when spawning subprocess
  • 876494fd967d cleanup: delete lots of unused local variables
  • 57264906a996 watchman: add the possibility to set the exact watchman binary location

The newly-vendored code has support for specifying the binary location,
so 57264906a996 does not need applied. But we do need to modify our
code to specify a proper argument name.

876494fd967d is not important, so it will be ignored.

c31ce080eb75 globally changed the code base to always pass
str to subprocess. But pywatchman's code is Python 3 clean, so
we don't need to do this.

This leaves dd35abc409ee and b1f62cd39b5c, which will be re-applied in
subsequent commits.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Nov 2 2019, 6:03 PM
indygreg added a commit: rHG6469c23a40a2: fsmonitor: refresh pywatchman with upstream.Nov 4 2019, 11:26 AM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG6469c23a40a2: fsmonitor: refresh pywatchman with upstream (authored by indygreg). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 17517

hgext/fsmonitor/pywatchman/__init__.py

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals # no unicode literals
from __future__ import absolute_import, division, print_function
import inspect import inspect
import math import math
import os import os
import socket import socket
import subprocess import subprocess
import time import time
from . import capabilities, compat, encoding, load
# Sometimes it's really hard to get Python extensions to compile, # Sometimes it's really hard to get Python extensions to compile,
# so fall back to a pure Python implementation. # so fall back to a pure Python implementation.
try: try:
from . import bser from . import bser
# Demandimport causes modules to be loaded lazily. Force the load now # Demandimport causes modules to be loaded lazily. Force the load now
# so that we can fall back on pybser if bser doesn't exist # so that we can fall back on pybser if bser doesn't exist
bser.pdu_info bser.pdu_info
except ImportError: except ImportError:
from . import pybser as bser from . import pybser as bser
from mercurial.utils import (
procutil,
)
from mercurial import (
pycompat,
)
from . import (
capabilities,
compat,
encoding,
load,
)
if os.name == 'nt': if os.name == "nt":
import ctypes import ctypes
import ctypes.wintypes import ctypes.wintypes
wintypes = ctypes.wintypes wintypes = ctypes.wintypes
GENERIC_READ = 0x80000000 GENERIC_READ = 0x80000000
GENERIC_WRITE = 0x40000000 GENERIC_WRITE = 0x40000000
FILE_FLAG_OVERLAPPED = 0x40000000 FILE_FLAG_OVERLAPPED = 0x40000000
OPEN_EXISTING = 3 OPEN_EXISTING = 3
INVALID_HANDLE_VALUE = -1 INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000 FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000
FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100 FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100
FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200 FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200
WAIT_FAILED = 0xFFFFFFFF WAIT_FAILED = 0xFFFFFFFF
WAIT_TIMEOUT = 0x00000102 WAIT_TIMEOUT = 0x00000102
WAIT_OBJECT_0 = 0x00000000 WAIT_OBJECT_0 = 0x00000000
WAIT_IO_COMPLETION = 0x000000C0 WAIT_IO_COMPLETION = 0x000000C0
INFINITE = 0xFFFFFFFF INFINITE = 0xFFFFFFFF
# Overlapped I/O operation is in progress. (997) # Overlapped I/O operation is in progress. (997)
ERROR_IO_PENDING = 0x000003E5 ERROR_IO_PENDING = 0x000003E5
# The pointer size follows the architecture # The pointer size follows the architecture
# We use WPARAM since this type is already conditionally defined # We use WPARAM since this type is already conditionally defined
ULONG_PTR = ctypes.wintypes.WPARAM ULONG_PTR = ctypes.wintypes.WPARAM
class OVERLAPPED(ctypes.Structure): class OVERLAPPED(ctypes.Structure):
_fields_ = [ _fields_ = [
("Internal", ULONG_PTR), ("InternalHigh", ULONG_PTR), ("Internal", ULONG_PTR),
("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD), ("InternalHigh", ULONG_PTR),
("hEvent", wintypes.HANDLE) ("Offset", wintypes.DWORD),
("OffsetHigh", wintypes.DWORD),
("hEvent", wintypes.HANDLE),
] ]
def __init__(self): def __init__(self):
self.Internal = 0 self.Internal = 0
self.InternalHigh = 0 self.InternalHigh = 0
self.Offset = 0 self.Offset = 0
self.OffsetHigh = 0 self.OffsetHigh = 0
self.hEvent = 0 self.hEvent = 0
LPDWORD = ctypes.POINTER(wintypes.DWORD) LPDWORD = ctypes.POINTER(wintypes.DWORD)
CreateFile = ctypes.windll.kernel32.CreateFileA CreateFile = ctypes.windll.kernel32.CreateFileA
CreateFile.argtypes = [wintypes.LPSTR, wintypes.DWORD, wintypes.DWORD, CreateFile.argtypes = [
wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD, wintypes.LPSTR,
wintypes.HANDLE] wintypes.DWORD,
wintypes.DWORD,
wintypes.LPVOID,
wintypes.DWORD,
wintypes.DWORD,
wintypes.HANDLE,
]
CreateFile.restype = wintypes.HANDLE CreateFile.restype = wintypes.HANDLE
CloseHandle = ctypes.windll.kernel32.CloseHandle CloseHandle = ctypes.windll.kernel32.CloseHandle
CloseHandle.argtypes = [wintypes.HANDLE] CloseHandle.argtypes = [wintypes.HANDLE]
CloseHandle.restype = wintypes.BOOL CloseHandle.restype = wintypes.BOOL
ReadFile = ctypes.windll.kernel32.ReadFile ReadFile = ctypes.windll.kernel32.ReadFile
ReadFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD, ReadFile.argtypes = [
LPDWORD, ctypes.POINTER(OVERLAPPED)] wintypes.HANDLE,
wintypes.LPVOID,
wintypes.DWORD,
LPDWORD,
ctypes.POINTER(OVERLAPPED),
]
ReadFile.restype = wintypes.BOOL ReadFile.restype = wintypes.BOOL
WriteFile = ctypes.windll.kernel32.WriteFile WriteFile = ctypes.windll.kernel32.WriteFile
WriteFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD, WriteFile.argtypes = [
LPDWORD, ctypes.POINTER(OVERLAPPED)] wintypes.HANDLE,
wintypes.LPVOID,
wintypes.DWORD,
LPDWORD,
ctypes.POINTER(OVERLAPPED),
]
WriteFile.restype = wintypes.BOOL WriteFile.restype = wintypes.BOOL
GetLastError = ctypes.windll.kernel32.GetLastError GetLastError = ctypes.windll.kernel32.GetLastError
GetLastError.argtypes = [] GetLastError.argtypes = []
GetLastError.restype = wintypes.DWORD GetLastError.restype = wintypes.DWORD
SetLastError = ctypes.windll.kernel32.SetLastError SetLastError = ctypes.windll.kernel32.SetLastError
SetLastError.argtypes = [wintypes.DWORD] SetLastError.argtypes = [wintypes.DWORD]
SetLastError.restype = None SetLastError.restype = None
FormatMessage = ctypes.windll.kernel32.FormatMessageA FormatMessage = ctypes.windll.kernel32.FormatMessageA
FormatMessage.argtypes = [wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD, FormatMessage.argtypes = [
wintypes.DWORD, ctypes.POINTER(wintypes.LPSTR), wintypes.DWORD,
wintypes.DWORD, wintypes.LPVOID] wintypes.LPVOID,
wintypes.DWORD,
wintypes.DWORD,
ctypes.POINTER(wintypes.LPSTR),
wintypes.DWORD,
wintypes.LPVOID,
]
FormatMessage.restype = wintypes.DWORD FormatMessage.restype = wintypes.DWORD
LocalFree = ctypes.windll.kernel32.LocalFree LocalFree = ctypes.windll.kernel32.LocalFree
GetOverlappedResult = ctypes.windll.kernel32.GetOverlappedResult GetOverlappedResult = ctypes.windll.kernel32.GetOverlappedResult
GetOverlappedResult.argtypes = [wintypes.HANDLE, GetOverlappedResult.argtypes = [
ctypes.POINTER(OVERLAPPED), LPDWORD, wintypes.HANDLE,
wintypes.BOOL] ctypes.POINTER(OVERLAPPED),
LPDWORD,
wintypes.BOOL,
]
GetOverlappedResult.restype = wintypes.BOOL GetOverlappedResult.restype = wintypes.BOOL
GetOverlappedResultEx = getattr(ctypes.windll.kernel32, GetOverlappedResultEx = getattr(
'GetOverlappedResultEx', None) ctypes.windll.kernel32, "GetOverlappedResultEx", None
)
if GetOverlappedResultEx is not None: if GetOverlappedResultEx is not None:
GetOverlappedResultEx.argtypes = [wintypes.HANDLE, GetOverlappedResultEx.argtypes = [
ctypes.POINTER(OVERLAPPED), LPDWORD, wintypes.HANDLE,
wintypes.DWORD, wintypes.BOOL] ctypes.POINTER(OVERLAPPED),
LPDWORD,
wintypes.DWORD,
wintypes.BOOL,
]
GetOverlappedResultEx.restype = wintypes.BOOL GetOverlappedResultEx.restype = wintypes.BOOL
WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx
WaitForSingleObjectEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL] WaitForSingleObjectEx.argtypes = [
wintypes.HANDLE,
wintypes.DWORD,
wintypes.BOOL,
]
WaitForSingleObjectEx.restype = wintypes.DWORD WaitForSingleObjectEx.restype = wintypes.DWORD
CreateEvent = ctypes.windll.kernel32.CreateEventA CreateEvent = ctypes.windll.kernel32.CreateEventA
CreateEvent.argtypes = [LPDWORD, wintypes.BOOL, wintypes.BOOL, CreateEvent.argtypes = [
wintypes.LPSTR] LPDWORD,
wintypes.BOOL,
wintypes.BOOL,
wintypes.LPSTR,
]
CreateEvent.restype = wintypes.HANDLE CreateEvent.restype = wintypes.HANDLE
# Windows Vista is the minimum supported client for CancelIoEx. # Windows Vista is the minimum supported client for CancelIoEx.
CancelIoEx = ctypes.windll.kernel32.CancelIoEx CancelIoEx = ctypes.windll.kernel32.CancelIoEx
CancelIoEx.argtypes = [wintypes.HANDLE, ctypes.POINTER(OVERLAPPED)] CancelIoEx.argtypes = [wintypes.HANDLE, ctypes.POINTER(OVERLAPPED)]
CancelIoEx.restype = wintypes.BOOL CancelIoEx.restype = wintypes.BOOL
# 2 bytes marker, 1 byte int size, 8 bytes int64 value # 2 bytes marker, 1 byte int size, 8 bytes int64 value
sniff_len = 13 sniff_len = 13
# This is a helper for debugging the client. # This is a helper for debugging the client.
_debugging = False _debugging = False
if _debugging: if _debugging:
def log(fmt, *args): def log(fmt, *args):
print('[%s] %s' % print(
(time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()), "[%s] %s"
fmt % args[:])) % (
time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()),
fmt % args[:],
)
)
else: else:
def log(fmt, *args): def log(fmt, *args):
pass pass
def _win32_strerror(err): def _win32_strerror(err):
""" expand a win32 error code into a human readable message """ """ expand a win32 error code into a human readable message """
# FormatMessage will allocate memory and assign it here # FormatMessage will allocate memory and assign it here
buf = ctypes.c_char_p() buf = ctypes.c_char_p()
FormatMessage( FormatMessage(
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER FORMAT_MESSAGE_FROM_SYSTEM
| FORMAT_MESSAGE_IGNORE_INSERTS, None, err, 0, buf, 0, None) | FORMAT_MESSAGE_ALLOCATE_BUFFER
| FORMAT_MESSAGE_IGNORE_INSERTS,
None,
err,
0,
buf,
0,
None,
)
try: try:
return buf.value return buf.value
finally: finally:
LocalFree(buf) LocalFree(buf)
class WatchmanError(Exception): class WatchmanError(Exception):
def __init__(self, msg=None, cmd=None): def __init__(self, msg=None, cmd=None):
self.msg = msg self.msg = msg
self.cmd = cmd self.cmd = cmd
def setCommand(self, cmd): def setCommand(self, cmd):
self.cmd = cmd self.cmd = cmd
def __str__(self): def __str__(self):
if self.cmd: if self.cmd:
return '%s, while executing %s' % (self.msg, self.cmd) return "%s, while executing %s" % (self.msg, self.cmd)
return self.msg return self.msg
class BSERv1Unsupported(WatchmanError):
pass
class UseAfterFork(WatchmanError):
pass
class WatchmanEnvironmentError(WatchmanError): class WatchmanEnvironmentError(WatchmanError):
def __init__(self, msg, errno, errmsg, cmd=None): def __init__(self, msg, errno, errmsg, cmd=None):
super(WatchmanEnvironmentError, self).__init__( super(WatchmanEnvironmentError, self).__init__(
'{0}: errno={1} errmsg={2}'.format(msg, errno, errmsg), "{0}: errno={1} errmsg={2}".format(msg, errno, errmsg), cmd
cmd) )
class SocketConnectError(WatchmanError): class SocketConnectError(WatchmanError):
def __init__(self, sockpath, exc): def __init__(self, sockpath, exc):
super(SocketConnectError, self).__init__( super(SocketConnectError, self).__init__(
'unable to connect to %s: %s' % (sockpath, exc)) "unable to connect to %s: %s" % (sockpath, exc)
)
self.sockpath = sockpath self.sockpath = sockpath
self.exc = exc self.exc = exc
class SocketTimeout(WatchmanError): class SocketTimeout(WatchmanError):
"""A specialized exception raised for socket timeouts during communication to/from watchman. """A specialized exception raised for socket timeouts during communication to/from watchman.
This makes it easier to implement non-blocking loops as callers can easily distinguish This makes it easier to implement non-blocking loops as callers can easily distinguish
between a routine timeout and an actual error condition. between a routine timeout and an actual error condition.
Note that catching WatchmanError will also catch this as it is a super-class, so backwards Note that catching WatchmanError will also catch this as it is a super-class, so backwards
compatibility in exception handling is preserved. compatibility in exception handling is preserved.
""" """
class CommandError(WatchmanError): class CommandError(WatchmanError):
"""error returned by watchman """error returned by watchman
self.msg is the message returned by watchman. self.msg is the message returned by watchman.
""" """
def __init__(self, msg, cmd=None): def __init__(self, msg, cmd=None):
super(CommandError, self).__init__( super(CommandError, self).__init__(
'watchman command error: %s' % (msg, ), "watchman command error: %s" % (msg,), cmd
cmd,
) )
class Transport(object): class Transport(object):
""" communication transport to the watchman server """ """ communication transport to the watchman server """
buf = None buf = None
def close(self): def close(self):
""" tear it down """ """ tear it down """
raise NotImplementedError() raise NotImplementedError()
def readBytes(self, size): def readBytes(self, size):
""" read size bytes """ """ read size bytes """
if len(self.buf) == 1 and b"\n" in self.buf[0]: if len(self.buf) == 1 and b"\n" in self.buf[0]:
(line, b) = self.buf[0].split(b"\n", 1) (line, b) = self.buf[0].split(b"\n", 1)
self.buf = [b] self.buf = [b]
return line return line
while True: while True:
b = self.readBytes(4096) b = self.readBytes(4096)
if b"\n" in b: if b"\n" in b:
result = b''.join(self.buf) result = b"".join(self.buf)
(line, b) = b.split(b"\n", 1) (line, b) = b.split(b"\n", 1)
self.buf = [b] self.buf = [b]
return result + line return result + line
self.buf.append(b) self.buf.append(b)
class Codec(object): class Codec(object):
""" communication encoding for the watchman server """ """ communication encoding for the watchman server """
transport = None transport = None
def __init__(self, transport): def __init__(self, transport):
self.transport = transport self.transport = transport
def receive(self): def receive(self):
raise NotImplementedError() raise NotImplementedError()
def send(self, *args): def send(self, *args):
raise NotImplementedError() raise NotImplementedError()
def setTimeout(self, value): def setTimeout(self, value):
self.transport.setTimeout(value) self.transport.setTimeout(value)
class UnixSocketTransport(Transport): class UnixSocketTransport(Transport):
""" local unix domain socket transport """ """ local unix domain socket transport """
sock = None sock = None
def __init__(self, sockpath, timeout, watchman_exe): def __init__(self, sockpath, timeout):
self.sockpath = sockpath self.sockpath = sockpath
self.timeout = timeout self.timeout = timeout
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try: try:
sock.settimeout(self.timeout) sock.settimeout(self.timeout)
sock.connect(self.sockpath) sock.connect(self.sockpath)
self.sock = sock self.sock = sock
except socket.error as e: except socket.error as e:
sock.close() sock.close()
raise SocketConnectError(self.sockpath, e) raise SocketConnectError(self.sockpath, e)
def close(self): def close(self):
if self.sock:
self.sock.close() self.sock.close()
self.sock = None self.sock = None
def setTimeout(self, value): def setTimeout(self, value):
self.timeout = value self.timeout = value
self.sock.settimeout(self.timeout) self.sock.settimeout(self.timeout)
def readBytes(self, size): def readBytes(self, size):
try: try:
buf = [self.sock.recv(size)] buf = [self.sock.recv(size)]
if not buf[0]: if not buf[0]:
raise WatchmanError('empty watchman response') raise WatchmanError("empty watchman response")
return buf[0] return buf[0]
except socket.timeout: except socket.timeout:
raise SocketTimeout('timed out waiting for response') raise SocketTimeout("timed out waiting for response")
def write(self, data): def write(self, data):
try: try:
self.sock.sendall(data) self.sock.sendall(data)
except socket.timeout: except socket.timeout:
raise SocketTimeout('timed out sending query command') raise SocketTimeout("timed out sending query command")
def _get_overlapped_result_ex_impl(pipe, olap, nbytes, millis, alertable): def _get_overlapped_result_ex_impl(pipe, olap, nbytes, millis, alertable):
""" Windows 7 and earlier does not support GetOverlappedResultEx. The """ Windows 7 and earlier does not support GetOverlappedResultEx. The
alternative is to use GetOverlappedResult and wait for read or write alternative is to use GetOverlappedResult and wait for read or write
operation to complete. This is done be using CreateEvent and operation to complete. This is done be using CreateEvent and
WaitForSingleObjectEx. CreateEvent, WaitForSingleObjectEx WaitForSingleObjectEx. CreateEvent, WaitForSingleObjectEx
and GetOverlappedResult are all part of Windows API since WindowsXP. and GetOverlappedResult are all part of Windows API since WindowsXP.
This is the exact same implementation that can be found in the watchman This is the exact same implementation that can be found in the watchman
source code (see get_overlapped_result_ex_impl in stream_win.c). This source code (see get_overlapped_result_ex_impl in stream_win.c). This
way, maintenance should be simplified. way, maintenance should be simplified.
""" """
log('Preparing to wait for maximum %dms', millis ) log("Preparing to wait for maximum %dms", millis)
if millis != 0: if millis != 0:
waitReturnCode = WaitForSingleObjectEx(olap.hEvent, millis, alertable) waitReturnCode = WaitForSingleObjectEx(olap.hEvent, millis, alertable)
if waitReturnCode == WAIT_OBJECT_0: if waitReturnCode == WAIT_OBJECT_0:
# Event is signaled, overlapped IO operation result should be available. # Event is signaled, overlapped IO operation result should be available.
pass pass
elif waitReturnCode == WAIT_IO_COMPLETION: elif waitReturnCode == WAIT_IO_COMPLETION:
# WaitForSingleObjectEx returnes because the system added an I/O completion # WaitForSingleObjectEx returnes because the system added an I/O completion
# routine or an asynchronous procedure call (APC) to the thread queue. # routine or an asynchronous procedure call (APC) to the thread queue.
SetLastError(WAIT_IO_COMPLETION) SetLastError(WAIT_IO_COMPLETION)
pass pass
elif waitReturnCode == WAIT_TIMEOUT: elif waitReturnCode == WAIT_TIMEOUT:
# We reached the maximum allowed wait time, the IO operation failed # We reached the maximum allowed wait time, the IO operation failed
# to complete in timely fashion. # to complete in timely fashion.
SetLastError(WAIT_TIMEOUT) SetLastError(WAIT_TIMEOUT)
return False return False
elif waitReturnCode == WAIT_FAILED: elif waitReturnCode == WAIT_FAILED:
# something went wrong calling WaitForSingleObjectEx # something went wrong calling WaitForSingleObjectEx
err = GetLastError() err = GetLastError()
log('WaitForSingleObjectEx failed: %s', _win32_strerror(err)) log("WaitForSingleObjectEx failed: %s", _win32_strerror(err))
return False return False
else: else:
# unexpected situation deserving investigation. # unexpected situation deserving investigation.
err = GetLastError() err = GetLastError()
log('Unexpected error: %s', _win32_strerror(err)) log("Unexpected error: %s", _win32_strerror(err))
return False return False
return GetOverlappedResult(pipe, olap, nbytes, False) return GetOverlappedResult(pipe, olap, nbytes, False)
class WindowsNamedPipeTransport(Transport): class WindowsNamedPipeTransport(Transport):
""" connect to a named pipe """ """ connect to a named pipe """
def __init__(self, sockpath, timeout, watchman_exe): def __init__(self, sockpath, timeout):
self.sockpath = sockpath self.sockpath = sockpath
self.timeout = int(math.ceil(timeout * 1000)) self.timeout = int(math.ceil(timeout * 1000))
self._iobuf = None self._iobuf = None
self.pipe = CreateFile(sockpath, GENERIC_READ | GENERIC_WRITE, 0, None, if compat.PYTHON3:
OPEN_EXISTING, FILE_FLAG_OVERLAPPED, None) sockpath = os.fsencode(sockpath)
self.pipe = CreateFile(
sockpath,
GENERIC_READ | GENERIC_WRITE,
0,
None,
OPEN_EXISTING,
FILE_FLAG_OVERLAPPED,
None,
)
if self.pipe == INVALID_HANDLE_VALUE: err = GetLastError()
if self.pipe == INVALID_HANDLE_VALUE or self.pipe == 0:
self.pipe = None self.pipe = None
self._raise_win_err('failed to open pipe %s' % sockpath, raise SocketConnectError(self.sockpath, self._make_win_err("", err))
GetLastError())
# event for the overlapped I/O operations # event for the overlapped I/O operations
self._waitable = CreateEvent(None, True, False, None) self._waitable = CreateEvent(None, True, False, None)
err = GetLastError()
if self._waitable is None: if self._waitable is None:
self._raise_win_err('CreateEvent failed', GetLastError()) self._raise_win_err("CreateEvent failed", err)
self._get_overlapped_result_ex = GetOverlappedResultEx self._get_overlapped_result_ex = GetOverlappedResultEx
if (os.getenv('WATCHMAN_WIN7_COMPAT') == '1' or if (
self._get_overlapped_result_ex is None): os.getenv("WATCHMAN_WIN7_COMPAT") == "1"
or self._get_overlapped_result_ex is None
):
self._get_overlapped_result_ex = _get_overlapped_result_ex_impl self._get_overlapped_result_ex = _get_overlapped_result_ex_impl
def _raise_win_err(self, msg, err): def _raise_win_err(self, msg, err):
raise IOError('%s win32 error code: %d %s' % raise self._make_win_err(msg, err)
(msg, err, _win32_strerror(err)))
def _make_win_err(self, msg, err):
return IOError(
"%s win32 error code: %d %s" % (msg, err, _win32_strerror(err))
)
def close(self): def close(self):
if self.pipe: if self.pipe:
log('Closing pipe') log("Closing pipe")
CloseHandle(self.pipe) CloseHandle(self.pipe)
self.pipe = None self.pipe = None
if self._waitable is not None: if self._waitable is not None:
# We release the handle for the event # We release the handle for the event
CloseHandle(self._waitable) CloseHandle(self._waitable)
self._waitable = None self._waitable = None
self._iobuf = self._iobuf[size:] self._iobuf = self._iobuf[size:]
return res return res
# We need to initiate a read # We need to initiate a read
buf = ctypes.create_string_buffer(size) buf = ctypes.create_string_buffer(size)
olap = OVERLAPPED() olap = OVERLAPPED()
olap.hEvent = self._waitable olap.hEvent = self._waitable
log('made read buff of size %d', size) log("made read buff of size %d", size)
# ReadFile docs warn against sending in the nread parameter for async # ReadFile docs warn against sending in the nread parameter for async
# operations, so we always collect it via GetOverlappedResultEx # operations, so we always collect it via GetOverlappedResultEx
immediate = ReadFile(self.pipe, buf, size, None, olap) immediate = ReadFile(self.pipe, buf, size, None, olap)
if not immediate: if not immediate:
err = GetLastError() err = GetLastError()
if err != ERROR_IO_PENDING: if err != ERROR_IO_PENDING:
self._raise_win_err('failed to read %d bytes' % size, self._raise_win_err("failed to read %d bytes" % size, err)
GetLastError())
nread = wintypes.DWORD() nread = wintypes.DWORD()
if not self._get_overlapped_result_ex(self.pipe, olap, nread, if not self._get_overlapped_result_ex(
0 if immediate else self.timeout, self.pipe, olap, nread, 0 if immediate else self.timeout, True
True): ):
err = GetLastError() err = GetLastError()
CancelIoEx(self.pipe, olap) CancelIoEx(self.pipe, olap)
if err == WAIT_TIMEOUT: if err == WAIT_TIMEOUT:
log('GetOverlappedResultEx timedout') log("GetOverlappedResultEx timedout")
raise SocketTimeout('timed out after waiting %dms for read' % raise SocketTimeout(
self.timeout) "timed out after waiting %dms for read" % self.timeout
)
log('GetOverlappedResultEx reports error %d', err) log("GetOverlappedResultEx reports error %d", err)
self._raise_win_err('error while waiting for read', err) self._raise_win_err("error while waiting for read", err)
nread = nread.value nread = nread.value
if nread == 0: if nread == 0:
# Docs say that named pipes return 0 byte when the other end did # Docs say that named pipes return 0 byte when the other end did
# a zero byte write. Since we don't ever do that, the only # a zero byte write. Since we don't ever do that, the only
# other way this shows up is if the client has gotten in a weird # other way this shows up is if the client has gotten in a weird
# state, so let's bail out # state, so let's bail out
CancelIoEx(self.pipe, olap) CancelIoEx(self.pipe, olap)
raise IOError('Async read yielded 0 bytes; unpossible!') raise IOError("Async read yielded 0 bytes; unpossible!")
# Holds precisely the bytes that we read from the prior request # Holds precisely the bytes that we read from the prior request
buf = buf[:nread] buf = buf[:nread]
returned_size = min(nread, size) returned_size = min(nread, size)
if returned_size == nread: if returned_size == nread:
return buf return buf
# keep any left-overs around for a later read to consume # keep any left-overs around for a later read to consume
self._iobuf = buf[returned_size:] self._iobuf = buf[returned_size:]
return buf[:returned_size] return buf[:returned_size]
def write(self, data): def write(self, data):
olap = OVERLAPPED() olap = OVERLAPPED()
olap.hEvent = self._waitable olap.hEvent = self._waitable
immediate = WriteFile(self.pipe, ctypes.c_char_p(data), len(data), immediate = WriteFile(
None, olap) self.pipe, ctypes.c_char_p(data), len(data), None, olap
)
if not immediate: if not immediate:
err = GetLastError() err = GetLastError()
if err != ERROR_IO_PENDING: if err != ERROR_IO_PENDING:
self._raise_win_err('failed to write %d bytes' % len(data), self._raise_win_err(
GetLastError()) "failed to write %d bytes to handle %r"
% (len(data), self.pipe),
err,
)
# Obtain results, waiting if needed # Obtain results, waiting if needed
nwrote = wintypes.DWORD() nwrote = wintypes.DWORD()
if self._get_overlapped_result_ex(self.pipe, olap, nwrote, if self._get_overlapped_result_ex(
0 if immediate else self.timeout, self.pipe, olap, nwrote, 0 if immediate else self.timeout, True
True): ):
log('made write of %d bytes', nwrote.value) log("made write of %d bytes", nwrote.value)
return nwrote.value return nwrote.value
err = GetLastError() err = GetLastError()
# It's potentially unsafe to allow the write to continue after # It's potentially unsafe to allow the write to continue after
# we unwind, so let's make a best effort to avoid that happening # we unwind, so let's make a best effort to avoid that happening
CancelIoEx(self.pipe, olap) CancelIoEx(self.pipe, olap)
if err == WAIT_TIMEOUT: if err == WAIT_TIMEOUT:
raise SocketTimeout('timed out after waiting %dms for write' % raise SocketTimeout(
self.timeout) "timed out after waiting %dms for write" % self.timeout
self._raise_win_err('error while waiting for write of %d bytes' % )
len(data), err) self._raise_win_err(
"error while waiting for write of %d bytes" % len(data), err
)
def _default_binpath(binpath=None):
if binpath:
return binpath
# The test harness sets WATCHMAN_BINARY to the binary under test,
# so we use that by default, otherwise, allow resolving watchman
# from the users PATH.
return os.environ.get("WATCHMAN_BINARY", "watchman")
class CLIProcessTransport(Transport): class CLIProcessTransport(Transport):
""" open a pipe to the cli to talk to the service """ open a pipe to the cli to talk to the service
This intended to be used only in the test harness! This intended to be used only in the test harness!
The CLI is an oddball because we only support JSON input The CLI is an oddball because we only support JSON input
and cannot send multiple commands through the same instance, and cannot send multiple commands through the same instance,
so we spawn a new process for each command. so we spawn a new process for each command.
We disable server spawning for this implementation, again, because We disable server spawning for this implementation, again, because
it is intended to be used only in our test harness. You really it is intended to be used only in our test harness. You really
should not need to use the CLI transport for anything real. should not need to use the CLI transport for anything real.
While the CLI can output in BSER, our Transport interface doesn't While the CLI can output in BSER, our Transport interface doesn't
support telling this instance that it should do so. That effectively support telling this instance that it should do so. That effectively
limits this implementation to JSON input and output only at this time. limits this implementation to JSON input and output only at this time.
It is the responsibility of the caller to set the send and It is the responsibility of the caller to set the send and
receive codecs appropriately. receive codecs appropriately.
""" """
proc = None proc = None
closed = True closed = True
def __init__(self, sockpath, timeout, watchman_exe): def __init__(self, sockpath, timeout, binpath=None):
self.sockpath = sockpath self.sockpath = sockpath
self.timeout = timeout self.timeout = timeout
self.watchman_exe = watchman_exe self.binpath = _default_binpath(binpath)
def close(self): def close(self):
if self.proc: if self.proc:
if self.proc.pid is not None: if self.proc.pid is not None:
self.proc.kill() self.proc.kill()
self.proc.stdin.close() self.proc.stdin.close()
self.proc.stdout.close() self.proc.stdout.close()
self.proc.wait()
self.proc = None self.proc = None
def _connect(self): def _connect(self):
if self.proc: if self.proc:
return self.proc return self.proc
args = [ args = [
self.watchman_exe, self.binpath,
'--sockname={0}'.format(self.sockpath), "--sockname={0}".format(self.sockpath),
'--logfile=/BOGUS', "--logfile=/BOGUS",
'--statefile=/BOGUS', "--statefile=/BOGUS",
'--no-spawn', "--no-spawn",
'--no-local', "--no-local",
'--no-pretty', "--no-pretty",
'-j', "-j",
] ]
self.proc = subprocess.Popen(pycompat.rapply(procutil.tonativestr, self.proc = subprocess.Popen(
args), args, stdin=subprocess.PIPE, stdout=subprocess.PIPE
stdin=subprocess.PIPE, )
stdout=subprocess.PIPE)
return self.proc return self.proc
def readBytes(self, size): def readBytes(self, size):
self._connect() self._connect()
res = self.proc.stdout.read(size) res = self.proc.stdout.read(size)
if res == '': if not res:
raise WatchmanError('EOF on CLI process transport') raise WatchmanError("EOF on CLI process transport")
return res return res
def write(self, data): def write(self, data):
if self.closed: if self.closed:
self.close() self.close()
self.closed = False self.closed = False
self._connect() self._connect()
res = self.proc.stdin.write(data) res = self.proc.stdin.write(data)
self.proc.stdin.close() self.proc.stdin.close()
self.closed = True self.closed = True
return res return res
class BserCodec(Codec): class BserCodec(Codec):
""" use the BSER encoding. This is the default, preferred codec """ """ use the BSER encoding. This is the default, preferred codec """
def __init__(self, transport, value_encoding, value_errors):
super(BserCodec, self).__init__(transport)
self._value_encoding = value_encoding
self._value_errors = value_errors
def _loads(self, response): def _loads(self, response):
return bser.loads(response) # Defaults to BSER v1 return bser.loads(
response,
value_encoding=self._value_encoding,
value_errors=self._value_errors,
)
def receive(self): def receive(self):
buf = [self.transport.readBytes(sniff_len)] buf = [self.transport.readBytes(sniff_len)]
if not buf[0]: if not buf[0]:
raise WatchmanError('empty watchman response') raise WatchmanError("empty watchman response")
_1, _2, elen = bser.pdu_info(buf[0]) _1, _2, elen = bser.pdu_info(buf[0])
rlen = len(buf[0]) rlen = len(buf[0])
while elen > rlen: while elen > rlen:
buf.append(self.transport.readBytes(elen - rlen)) buf.append(self.transport.readBytes(elen - rlen))
rlen += len(buf[-1]) rlen += len(buf[-1])
response = b''.join(buf) response = b"".join(buf)
try: try:
res = self._loads(response) res = self._loads(response)
return res return res
except ValueError as e: except ValueError as e:
raise WatchmanError('watchman response decode error: %s' % e) raise WatchmanError("watchman response decode error: %s" % e)
def send(self, *args): def send(self, *args):
cmd = bser.dumps(*args) # Defaults to BSER v1 cmd = bser.dumps(*args) # Defaults to BSER v1
self.transport.write(cmd) self.transport.write(cmd)
class ImmutableBserCodec(BserCodec): class ImmutableBserCodec(BserCodec):
""" use the BSER encoding, decoding values using the newer """ use the BSER encoding, decoding values using the newer
immutable object support """ immutable object support """
def _loads(self, response): def _loads(self, response):
return bser.loads(response, False) # Defaults to BSER v1 return bser.loads(
response,
False,
value_encoding=self._value_encoding,
value_errors=self._value_errors,
)
class Bser2WithFallbackCodec(BserCodec): class Bser2WithFallbackCodec(BserCodec):
""" use BSER v2 encoding """ """ use BSER v2 encoding """
def __init__(self, transport): def __init__(self, transport, value_encoding, value_errors):
super(Bser2WithFallbackCodec, self).__init__(transport) super(Bser2WithFallbackCodec, self).__init__(
# Once the server advertises support for bser-v2 we should switch this transport, value_encoding, value_errors
# to 'required' on Python 3. )
self.send(["version", {"optional": ["bser-v2"]}]) if compat.PYTHON3:
bserv2_key = "required"
else:
bserv2_key = "optional"
self.send(["version", {bserv2_key: ["bser-v2"]}])
capabilities = self.receive() capabilities = self.receive()
if 'error' in capabilities: if "error" in capabilities:
raise Exception('Unsupported BSER version') raise BSERv1Unsupported(
"The watchman server version does not support Python 3. Please "
"upgrade your watchman server."
)
if capabilities['capabilities']['bser-v2']: if capabilities["capabilities"]["bser-v2"]:
self.bser_version = 2 self.bser_version = 2
self.bser_capabilities = 0 self.bser_capabilities = 0
else: else:
self.bser_version = 1 self.bser_version = 1
self.bser_capabilities = 0 self.bser_capabilities = 0
def _loads(self, response):
return bser.loads(response)
def receive(self): def receive(self):
buf = [self.transport.readBytes(sniff_len)] buf = [self.transport.readBytes(sniff_len)]
if not buf[0]: if not buf[0]:
raise WatchmanError('empty watchman response') raise WatchmanError("empty watchman response")
recv_bser_version, recv_bser_capabilities, elen = bser.pdu_info(buf[0]) recv_bser_version, recv_bser_capabilities, elen = bser.pdu_info(buf[0])
if hasattr(self, 'bser_version'): if hasattr(self, "bser_version"):
# Readjust BSER version and capabilities if necessary # Readjust BSER version and capabilities if necessary
self.bser_version = max(self.bser_version, recv_bser_version) self.bser_version = max(self.bser_version, recv_bser_version)
self.capabilities = self.bser_capabilities & recv_bser_capabilities self.capabilities = self.bser_capabilities & recv_bser_capabilities
rlen = len(buf[0]) rlen = len(buf[0])
while elen > rlen: while elen > rlen:
buf.append(self.transport.readBytes(elen - rlen)) buf.append(self.transport.readBytes(elen - rlen))
rlen += len(buf[-1]) rlen += len(buf[-1])
response = b''.join(buf) response = b"".join(buf)
try: try:
res = self._loads(response) res = self._loads(response)
return res return res
except ValueError as e: except ValueError as e:
raise WatchmanError('watchman response decode error: %s' % e) raise WatchmanError("watchman response decode error: %s" % e)
def send(self, *args): def send(self, *args):
if hasattr(self, 'bser_version'): if hasattr(self, "bser_version"):
cmd = bser.dumps(*args, version=self.bser_version, cmd = bser.dumps(
capabilities=self.bser_capabilities) *args,
version=self.bser_version,
capabilities=self.bser_capabilities
)
else: else:
cmd = bser.dumps(*args) cmd = bser.dumps(*args)
self.transport.write(cmd) self.transport.write(cmd)
class ImmutableBser2Codec(Bser2WithFallbackCodec, ImmutableBserCodec):
""" use the BSER encoding, decoding values using the newer
immutable object support """
pass
class JsonCodec(Codec): class JsonCodec(Codec):
""" Use json codec. This is here primarily for testing purposes """ """ Use json codec. This is here primarily for testing purposes """
json = None json = None
def __init__(self, transport): def __init__(self, transport):
super(JsonCodec, self).__init__(transport) super(JsonCodec, self).__init__(transport)
# optional dep on json, only if JsonCodec is used # optional dep on json, only if JsonCodec is used
import json import json
self.json = json self.json = json
def receive(self): def receive(self):
line = self.transport.readLine() line = self.transport.readLine()
try: try:
# In Python 3, json.loads is a transformation from Unicode string to # In Python 3, json.loads is a transformation from Unicode string to
# objects possibly containing Unicode strings. We typically expect # objects possibly containing Unicode strings. We typically expect
# the JSON blob to be ASCII-only with non-ASCII characters escaped, # the JSON blob to be ASCII-only with non-ASCII characters escaped,
# but it's possible we might get non-ASCII bytes that are valid # but it's possible we might get non-ASCII bytes that are valid
# UTF-8. # UTF-8.
if compat.PYTHON3: if compat.PYTHON3:
line = line.decode('utf-8') line = line.decode("utf-8")
return self.json.loads(line) return self.json.loads(line)
except Exception as e: except Exception as e:
print(e, line) print(e, line)
raise raise
def send(self, *args): def send(self, *args):
cmd = self.json.dumps(*args) cmd = self.json.dumps(*args)
# In Python 3, json.dumps is a transformation from objects possibly # In Python 3, json.dumps is a transformation from objects possibly
# containing Unicode strings to Unicode string. Even with (the default) # containing Unicode strings to Unicode string. Even with (the default)
# ensure_ascii=True, dumps returns a Unicode string. # ensure_ascii=True, dumps returns a Unicode string.
if compat.PYTHON3: if compat.PYTHON3:
cmd = cmd.encode('ascii') cmd = cmd.encode("ascii")
self.transport.write(cmd + b"\n") self.transport.write(cmd + b"\n")
class client(object): class client(object):
""" Handles the communication with the watchman service """ """ Handles the communication with the watchman service """
sockpath = None sockpath = None
transport = None transport = None
sendCodec = None sendCodec = None
recvCodec = None recvCodec = None
sendConn = None sendConn = None
recvConn = None recvConn = None
subs = {} # Keyed by subscription name subs = {} # Keyed by subscription name
sub_by_root = {} # Keyed by root, then by subscription name sub_by_root = {} # Keyed by root, then by subscription name
logs = [] # When log level is raised logs = [] # When log level is raised
unilateral = ['log', 'subscription'] unilateral = ["log", "subscription"]
tport = None tport = None
useImmutableBser = None useImmutableBser = None
watchman_exe = None pid = None
def __init__(self, def __init__(
self,
sockpath=None, sockpath=None,
timeout=1.0, timeout=1.0,
transport=None, transport=None,
sendEncoding=None, sendEncoding=None,
recvEncoding=None, recvEncoding=None,
useImmutableBser=False, useImmutableBser=False,
watchman_exe=None): # use False for these two because None has a special
# meaning
valueEncoding=False,
valueErrors=False,
binpath=None,
):
self.sockpath = sockpath self.sockpath = sockpath
self.timeout = timeout self.timeout = timeout
self.useImmutableBser = useImmutableBser self.useImmutableBser = useImmutableBser
self.watchman_exe = watchman_exe self.binpath = _default_binpath(binpath)
if inspect.isclass(transport) and issubclass(transport, Transport): if inspect.isclass(transport) and issubclass(transport, Transport):
self.transport = transport self.transport = transport
else: else:
transport = transport or os.getenv('WATCHMAN_TRANSPORT') or 'local' transport = transport or os.getenv("WATCHMAN_TRANSPORT") or "local"
if transport == 'local' and os.name == 'nt': if transport == "local" and os.name == "nt":
self.transport = WindowsNamedPipeTransport self.transport = WindowsNamedPipeTransport
elif transport == 'local': elif transport == "local":
self.transport = UnixSocketTransport self.transport = UnixSocketTransport
elif transport == 'cli': elif transport == "cli":
self.transport = CLIProcessTransport self.transport = CLIProcessTransport
if sendEncoding is None: if sendEncoding is None:
sendEncoding = 'json' sendEncoding = "json"
if recvEncoding is None: if recvEncoding is None:
recvEncoding = sendEncoding recvEncoding = sendEncoding
else: else:
raise WatchmanError('invalid transport %s' % transport) raise WatchmanError("invalid transport %s" % transport)
sendEncoding = str(sendEncoding or os.getenv('WATCHMAN_ENCODING') or sendEncoding = str(
'bser') sendEncoding or os.getenv("WATCHMAN_ENCODING") or "bser"
recvEncoding = str(recvEncoding or os.getenv('WATCHMAN_ENCODING') or )
'bser') recvEncoding = str(
recvEncoding or os.getenv("WATCHMAN_ENCODING") or "bser"
)
self.recvCodec = self._parseEncoding(recvEncoding) self.recvCodec = self._parseEncoding(recvEncoding)
self.sendCodec = self._parseEncoding(sendEncoding) self.sendCodec = self._parseEncoding(sendEncoding)
# We want to act like the native OS methods as much as possible. This
# means returning bytestrings on Python 2 by default and Unicode
# strings on Python 3. However we take an optional argument that lets
# users override this.
if valueEncoding is False:
if compat.PYTHON3:
self.valueEncoding = encoding.get_local_encoding()
self.valueErrors = encoding.default_local_errors
else:
self.valueEncoding = None
self.valueErrors = None
else:
self.valueEncoding = valueEncoding
if valueErrors is False:
self.valueErrors = encoding.default_local_errors
else:
self.valueErrors = valueErrors
def _makeBSERCodec(self, codec):
def make_codec(transport):
return codec(transport, self.valueEncoding, self.valueErrors)
return make_codec
def _parseEncoding(self, enc): def _parseEncoding(self, enc):
if enc == 'bser': if enc == "bser":
if self.useImmutableBser:
return self._makeBSERCodec(ImmutableBser2Codec)
return self._makeBSERCodec(Bser2WithFallbackCodec)
elif enc == "bser-v1":
if compat.PYTHON3:
raise BSERv1Unsupported(
"Python 3 does not support the BSER v1 encoding: specify "
'"bser" or omit the sendEncoding and recvEncoding '
"arguments"
)
if self.useImmutableBser: if self.useImmutableBser:
return ImmutableBserCodec return self._makeBSERCodec(ImmutableBserCodec)
return BserCodec return self._makeBSERCodec(BserCodec)
elif enc == 'experimental-bser-v2': elif enc == "json":
return Bser2WithFallbackCodec
elif enc == 'json':
return JsonCodec return JsonCodec
else: else:
raise WatchmanError('invalid encoding %s' % enc) raise WatchmanError("invalid encoding %s" % enc)
def _hasprop(self, result, name): def _hasprop(self, result, name):
if self.useImmutableBser: if self.useImmutableBser:
return hasattr(result, name) return hasattr(result, name)
return name in result return name in result
def _resolvesockname(self): def _resolvesockname(self):
# if invoked via a trigger, watchman will set this env var; we # if invoked via a trigger, watchman will set this env var; we
# should use it unless explicitly set otherwise # should use it unless explicitly set otherwise
path = os.getenv('WATCHMAN_SOCK') path = os.getenv("WATCHMAN_SOCK")
if path: if path:
return path return path
cmd = [self.watchman_exe, '--output-encoding=bser', 'get-sockname'] cmd = [self.binpath, "--output-encoding=bser", "get-sockname"]
try: try:
args = dict(stdout=subprocess.PIPE, args = dict(
stderr=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
close_fds=os.name != 'nt') ) # noqa: C408
if os.name == 'nt': if os.name == "nt":
# if invoked via an application with graphical user interface, # if invoked via an application with graphical user interface,
# this call will cause a brief command window pop-up. # this call will cause a brief command window pop-up.
# Using the flag STARTF_USESHOWWINDOW to avoid this behavior. # Using the flag STARTF_USESHOWWINDOW to avoid this behavior.
startupinfo = subprocess.STARTUPINFO() startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
args['startupinfo'] = startupinfo args["startupinfo"] = startupinfo
p = subprocess.Popen(pycompat.rapply(procutil.tonativestr, cmd), p = subprocess.Popen(cmd, **args)
**args)
except OSError as e: except OSError as e:
raise WatchmanError('"watchman" executable not in PATH (%s)' % e) raise WatchmanError('"watchman" executable not in PATH (%s)', e)
stdout, stderr = p.communicate() stdout, stderr = p.communicate()
exitcode = p.poll() exitcode = p.poll()
if exitcode: if exitcode:
raise WatchmanError("watchman exited with code %d" % exitcode) raise WatchmanError("watchman exited with code %d" % exitcode)
result = bser.loads(stdout) result = bser.loads(stdout)
if b'error' in result: if "error" in result:
raise WatchmanError('get-sockname error: %s' % result['error']) raise WatchmanError("get-sockname error: %s" % result["error"])
return result[b'sockname'] return result["sockname"]
def _connect(self): def _connect(self):
""" establish transport connection """ """ establish transport connection """
if self.recvConn: if self.recvConn:
if self.pid != os.getpid():
raise UseAfterFork(
"do not re-use a connection after fork; open a new client instead"
)
return return
if self.sockpath is None: if self.sockpath is None:
self.sockpath = self._resolvesockname() self.sockpath = self._resolvesockname()
self.tport = self.transport(self.sockpath, self.timeout, self.watchman_exe) kwargs = {}
if self.transport == CLIProcessTransport:
kwargs["binpath"] = self.binpath
self.tport = self.transport(self.sockpath, self.timeout, **kwargs)
self.sendConn = self.sendCodec(self.tport) self.sendConn = self.sendCodec(self.tport)
self.recvConn = self.recvCodec(self.tport) self.recvConn = self.recvCodec(self.tport)
self.pid = os.getpid()
def __del__(self): def __del__(self):
self.close() self.close()
def __enter__(self):
self._connect()
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
self.close()
def close(self): def close(self):
if self.tport: if self.tport:
self.tport.close() self.tport.close()
self.tport = None self.tport = None
self.recvConn = None self.recvConn = None
self.sendConn = None self.sendConn = None
def receive(self): def receive(self):
""" receive the next PDU from the watchman service """ receive the next PDU from the watchman service
If the client has activated subscriptions or logs then If the client has activated subscriptions or logs then
this PDU may be a unilateral PDU sent by the service to this PDU may be a unilateral PDU sent by the service to
inform the client of a log event or subscription change. inform the client of a log event or subscription change.
It may also simply be the response portion of a request It may also simply be the response portion of a request
initiated by query. initiated by query.
There are clients in production that subscribe and call There are clients in production that subscribe and call
this in a loop to retrieve all subscription responses, this in a loop to retrieve all subscription responses,
so care should be taken when making changes here. so care should be taken when making changes here.
""" """
self._connect() self._connect()
result = self.recvConn.receive() result = self.recvConn.receive()
if self._hasprop(result, 'error'): if self._hasprop(result, "error"):
error = result['error'] raise CommandError(result["error"])
if compat.PYTHON3 and isinstance(self.recvConn, BserCodec):
error = result['error'].decode('utf-8', 'surrogateescape')
raise CommandError(error)
if self._hasprop(result, 'log'):
log = result['log']
if compat.PYTHON3 and isinstance(self.recvConn, BserCodec):
log = log.decode('utf-8', 'surrogateescape')
self.logs.append(log)
if self._hasprop(result, 'subscription'): if self._hasprop(result, "log"):
sub = result['subscription'] self.logs.append(result["log"])
if self._hasprop(result, "subscription"):
sub = result["subscription"]
if not (sub in self.subs): if not (sub in self.subs):
self.subs[sub] = [] self.subs[sub] = []
self.subs[sub].append(result) self.subs[sub].append(result)
# also accumulate in {root,sub} keyed store # also accumulate in {root,sub} keyed store
root = os.path.normcase(result['root']) root = os.path.normpath(os.path.normcase(result["root"]))
if not root in self.sub_by_root: if not root in self.sub_by_root:
self.sub_by_root[root] = {} self.sub_by_root[root] = {}
if not sub in self.sub_by_root[root]: if not sub in self.sub_by_root[root]:
self.sub_by_root[root][sub] = [] self.sub_by_root[root][sub] = []
self.sub_by_root[root][sub].append(result) self.sub_by_root[root][sub].append(result)
return result return result
def isUnilateralResponse(self, res): def isUnilateralResponse(self, res):
if 'unilateral' in res and res['unilateral']: if "unilateral" in res and res["unilateral"]:
return True return True
# Fall back to checking for known unilateral responses # Fall back to checking for known unilateral responses
for k in self.unilateral: for k in self.unilateral:
if k in res: if k in res:
return True return True
return False return False
def getLog(self, remove=True): def getLog(self, remove=True):
Returns None if there is no data associated with `name` Returns None if there is no data associated with `name`
If root is not None, then only return the subscription If root is not None, then only return the subscription
data that matches both root and name. When used in this way, data that matches both root and name. When used in this way,
remove processing impacts both the unscoped and scoped stores remove processing impacts both the unscoped and scoped stores
for the subscription data. for the subscription data.
""" """
if compat.PYTHON3 and issubclass(self.recvCodec, BserCodec):
# People may pass in Unicode strings here -- but currently BSER only
# returns bytestrings. Deal with that.
if isinstance(root, str):
root = encoding.encode_local(root)
if isinstance(name, str):
name = name.encode('utf-8')
if root is not None: if root is not None:
if not root in self.sub_by_root: root = os.path.normpath(os.path.normcase(root))
if root not in self.sub_by_root:
return None return None
if not name in self.sub_by_root[root]: if name not in self.sub_by_root[root]:
return None return None
sub = self.sub_by_root[root][name] sub = self.sub_by_root[root][name]
if remove: if remove:
del self.sub_by_root[root][name] del self.sub_by_root[root][name]
# don't let this grow unbounded # don't let this grow unbounded
if name in self.subs: if name in self.subs:
del self.subs[name] del self.subs[name]
return sub return sub
if not (name in self.subs): if name not in self.subs:
return None return None
sub = self.subs[name] sub = self.subs[name]
if remove: if remove:
del self.subs[name] del self.subs[name]
return sub return sub
def query(self, *args): def query(self, *args):
""" Send a query to the watchman service and return the response """ Send a query to the watchman service and return the response
This call will block until the response is returned. This call will block until the response is returned.
If any unilateral responses are sent by the service in between If any unilateral responses are sent by the service in between
the request-response they will be buffered up in the client object the request-response they will be buffered up in the client object
and NOT returned via this method. and NOT returned via this method.
""" """
log('calling client.query') log("calling client.query")
self._connect() self._connect()
try: try:
self.sendConn.send(args) self.sendConn.send(args)
res = self.receive() res = self.receive()
while self.isUnilateralResponse(res): while self.isUnilateralResponse(res):
res = self.receive() res = self.receive()
return res return res
except EnvironmentError as ee: except EnvironmentError as ee:
# When we can depend on Python 3, we can use PEP 3134 # When we can depend on Python 3, we can use PEP 3134
# exception chaining here. # exception chaining here.
raise WatchmanEnvironmentError( raise WatchmanEnvironmentError(
'I/O error communicating with watchman daemon', "I/O error communicating with watchman daemon",
ee.errno, ee.errno,
ee.strerror, ee.strerror,
args) args,
)
except WatchmanError as ex: except WatchmanError as ex:
ex.setCommand(args) ex.setCommand(args)
raise raise
def capabilityCheck(self, optional=None, required=None): def capabilityCheck(self, optional=None, required=None):
""" Perform a server capability check """ """ Perform a server capability check """
res = self.query('version', { res = self.query(
'optional': optional or [], "version", {"optional": optional or [], "required": required or []}
'required': required or [] )
})
if not self._hasprop(res, 'capabilities'): if not self._hasprop(res, "capabilities"):
# Server doesn't support capabilities, so we need to # Server doesn't support capabilities, so we need to
# synthesize the results based on the version # synthesize the results based on the version
capabilities.synthesize(res, optional) capabilities.synthesize(res, optional)
if 'error' in res: if "error" in res:
raise CommandError(res['error']) raise CommandError(res["error"])
return res return res
def setTimeout(self, value): def setTimeout(self, value):
self.recvConn.setTimeout(value) self.recvConn.setTimeout(value)
self.sendConn.setTimeout(value) self.sendConn.setTimeout(value)

hgext/fsmonitor/pywatchman/bser.c

PyObject_Del(o); PyObject_Del(o);
} }
static PyObject* bserobj_getattrro(PyObject* o, PyObject* name) { static PyObject* bserobj_getattrro(PyObject* o, PyObject* name) {
bserObject* obj = (bserObject*)o; bserObject* obj = (bserObject*)o;
Py_ssize_t i, n; Py_ssize_t i, n;
PyObject* name_bytes = NULL; PyObject* name_bytes = NULL;
PyObject* ret = NULL; PyObject* ret = NULL;
const char* namestr = NULL; const char* namestr;
if (PyIndex_Check(name)) { if (PyIndex_Check(name)) {
i = PyNumber_AsSsize_t(name, PyExc_IndexError); i = PyNumber_AsSsize_t(name, PyExc_IndexError);
if (i == -1 && PyErr_Occurred()) { if (i == -1 && PyErr_Occurred()) {
goto bail; goto bail;
} }
if (i == 8 && PySequence_Size(obj->values) < 9) {
// Hack alert: Python 3 removed support for os.stat().st_mtime
// being an integer.Instead, if you need an integer, you have to
// use os.stat()[stat.ST_MTIME] instead. stat.ST_MTIME is 8, and
// our stat tuples are shorter than that, so we can detect
// requests for index 8 on tuples shorter than that and return
// st_mtime instead.
namestr = "st_mtime";
} else {
ret = PySequence_GetItem(obj->values, i); ret = PySequence_GetItem(obj->values, i);
goto bail; goto bail;
} }
} else {
// We can be passed in Unicode objects here -- we don't support anything other // We can be passed in Unicode objects here -- we don't support anything other
// than UTF-8 for keys. // than UTF-8 for keys.
if (PyUnicode_Check(name)) { if (PyUnicode_Check(name)) {
name_bytes = PyUnicode_AsUTF8String(name); name_bytes = PyUnicode_AsUTF8String(name);
if (name_bytes == NULL) { if (name_bytes == NULL) {
goto bail; goto bail;
} }
namestr = PyBytes_AsString(name_bytes); namestr = PyBytes_AsString(name_bytes);
} else { } else {
namestr = PyBytes_AsString(name); namestr = PyBytes_AsString(name);
} }
}
if (namestr == NULL) { if (namestr == NULL) {
goto bail; goto bail;
} }
// hack^Wfeature to allow mercurial to use "st_size" to reference "size" // hack^Wfeature to allow mercurial to use "st_size" to reference "size"
if (!strncmp(namestr, "st_", 3)) { if (!strncmp(namestr, "st_", 3)) {
namestr += 3; namestr += 3;
} }
PyErr_SetString(PyExc_ValueError, "bser data len != header len"); PyErr_SetString(PyExc_ValueError, "bser data len != header len");
return NULL; return NULL;
} }
return bser_loads_recursive(&data, end, &ctx); return bser_loads_recursive(&data, end, &ctx);
} }
static PyObject* bser_load(PyObject* self, PyObject* args, PyObject* kw) { static PyObject* bser_load(PyObject* self, PyObject* args, PyObject* kw) {
PyObject *load, *string; PyObject* load;
PyObject* load_method;
PyObject* string;
PyObject* load_method_args;
PyObject* load_method_kwargs;
PyObject* fp = NULL; PyObject* fp = NULL;
PyObject* mutable_obj = NULL; PyObject* mutable_obj = NULL;
const char* value_encoding = NULL; PyObject* value_encoding = NULL;
const char* value_errors = NULL; PyObject* value_errors = NULL;
static char* kw_list[] = { static char* kw_list[] = {
"fp", "mutable", "value_encoding", "value_errors", NULL}; "fp", "mutable", "value_encoding", "value_errors", NULL};
if (!PyArg_ParseTupleAndKeywords( if (!PyArg_ParseTupleAndKeywords(
args, args,
kw, kw,
"OOzz:load", "O|OOO:load",
kw_list, kw_list,
&fp, &fp,
&mutable_obj, &mutable_obj,
&value_encoding, &value_encoding,
&value_errors)) { &value_errors)) {
return NULL; return NULL;
} }
load = PyImport_ImportModule("pywatchman.load"); load = PyImport_ImportModule("pywatchman.load");
if (load == NULL) { if (load == NULL) {
return NULL; return NULL;
} }
string = PyObject_CallMethod( load_method = PyObject_GetAttrString(load, "load");
load, "load", "OOzz", fp, mutable_obj, value_encoding, value_errors); if (load_method == NULL) {
return NULL;
}
// Mandatory method arguments
load_method_args = Py_BuildValue("(O)", fp);
if (load_method_args == NULL) {
return NULL;
}
// Optional method arguments
load_method_kwargs = PyDict_New();
if (load_method_kwargs == NULL) {
return NULL;
}
if (mutable_obj) {
PyDict_SetItemString(load_method_kwargs, "mutable", mutable_obj);
}
if (value_encoding) {
PyDict_SetItemString(load_method_kwargs, "value_encoding", value_encoding);
}
if (value_errors) {
PyDict_SetItemString(load_method_kwargs, "value_errors", value_errors);
}
string = PyObject_Call(load_method, load_method_args, load_method_kwargs);
Py_DECREF(load_method_kwargs);
Py_DECREF(load_method_args);
Py_DECREF(load_method);
Py_DECREF(load); Py_DECREF(load);
return string; return string;
} }
// clang-format off // clang-format off
static PyMethodDef bser_methods[] = { static PyMethodDef bser_methods[] = {
{"loads", (PyCFunction)bser_loads, METH_VARARGS | METH_KEYWORDS, {"loads", (PyCFunction)bser_loads, METH_VARARGS | METH_KEYWORDS,
"Deserialize string."}, "Deserialize string."},

hgext/fsmonitor/pywatchman/capabilities.py

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals # no unicode literals
from __future__ import absolute_import, division, print_function
import re import re
def parse_version(vstr): def parse_version(vstr):
res = 0 res = 0
for n in vstr.split('.'): for n in vstr.split("."):
res = res * 1000 res = res * 1000
res = res + int(n) res = res + int(n)
return res return res
cap_versions = { cap_versions = {
"cmd-watch-del-all": "3.1.1", "cmd-watch-del-all": "3.1.1",
"cmd-watch-project": "3.1", "cmd-watch-project": "3.1",
"relative_root": "3.3", "relative_root": "3.3",
"term-dirname": "3.1", "term-dirname": "3.1",
"term-idirname": "3.1", "term-idirname": "3.1",
"wildmatch": "3.7", "wildmatch": "3.7",
} }
def check(version, name): def check(version, name):
if name in cap_versions: if name in cap_versions:
return version >= parse_version(cap_versions[name]) return version >= parse_version(cap_versions[name])
return False return False
def synthesize(vers, opts): def synthesize(vers, opts):
""" Synthesize a capability enabled version response """ Synthesize a capability enabled version response
This is a very limited emulation for relatively recent feature sets This is a very limited emulation for relatively recent feature sets
""" """
parsed_version = parse_version(vers['version']) parsed_version = parse_version(vers["version"])
vers['capabilities'] = {} vers["capabilities"] = {}
for name in opts['optional']: for name in opts["optional"]:
vers['capabilities'][name] = check(parsed_version, name) vers["capabilities"][name] = check(parsed_version, name)
for name in opts['required']: failed = False # noqa: F841 T25377293 Grandfathered in
for name in opts["required"]:
have = check(parsed_version, name) have = check(parsed_version, name)
vers['capabilities'][name] = have vers["capabilities"][name] = have
if not have: if not have:
vers['error'] = 'client required capability `' + name + \ vers["error"] = (
'` is not supported by this server' "client required capability `"
+ name
+ "` is not supported by this server"
)
return vers return vers

hgext/fsmonitor/pywatchman/compat.py

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals # no unicode literals
from __future__ import absolute_import, division, print_function
'''Compatibility module across Python 2 and 3.'''
import sys import sys
"""Compatibility module across Python 2 and 3."""
PYTHON2 = sys.version_info < (3, 0)
PYTHON3 = sys.version_info >= (3, 0) PYTHON3 = sys.version_info >= (3, 0)
# This is adapted from https://bitbucket.org/gutworth/six, and used under the # This is adapted from https://bitbucket.org/gutworth/six, and used under the
# MIT license. See LICENSE for a full copyright notice. # MIT license. See LICENSE for a full copyright notice.
if PYTHON3: if PYTHON3:
def reraise(tp, value, tb=None): def reraise(tp, value, tb=None):
try: try:
if value is None: if value is None:
value = tp() value = tp()
if value.__traceback__ is not tb: if value.__traceback__ is not tb:
raise value.with_traceback(tb) raise value.with_traceback(tb)
raise value raise value
finally: finally:
value = None value = None
tb = None tb = None
else: else:
exec(''' exec(
"""
def reraise(tp, value, tb=None): def reraise(tp, value, tb=None):
try: try:
raise tp, value, tb raise tp, value, tb
finally: finally:
tb = None tb = None
'''.strip()) """.strip()
)
if PYTHON3: if PYTHON3:
UNICODE = str UNICODE = str
else: else:
UNICODE = unicode UNICODE = unicode # noqa: F821 We handled versioning above

hgext/fsmonitor/pywatchman/encoding.py

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals # no unicode literals
from __future__ import absolute_import, division, print_function
'''Module to deal with filename encoding on the local system, as returned by
Watchman.'''
import sys import sys
from . import ( from . import compat
compat,
)
"""Module to deal with filename encoding on the local system, as returned by
Watchman."""
if compat.PYTHON3: if compat.PYTHON3:
default_local_errors = 'surrogateescape' default_local_errors = "surrogateescape"
def get_local_encoding(): def get_local_encoding():
if sys.platform == 'win32': if sys.platform == "win32":
# Watchman always returns UTF-8 encoded strings on Windows. # Watchman always returns UTF-8 encoded strings on Windows.
return 'utf-8' return "utf-8"
# On the Python 3 versions we support, sys.getfilesystemencoding never # On the Python 3 versions we support, sys.getfilesystemencoding never
# returns None. # returns None.
return sys.getfilesystemencoding() return sys.getfilesystemencoding()
else: else:
# Python 2 doesn't support surrogateescape, so use 'strict' by # Python 2 doesn't support surrogateescape, so use 'strict' by
# default. Users can register a custom surrogateescape error handler and use # default. Users can register a custom surrogateescape error handler and use
# that if they so desire. # that if they so desire.
default_local_errors = 'strict' default_local_errors = "strict"
def get_local_encoding(): def get_local_encoding():
if sys.platform == 'win32': if sys.platform == "win32":
# Watchman always returns UTF-8 encoded strings on Windows. # Watchman always returns UTF-8 encoded strings on Windows.
return 'utf-8' return "utf-8"
fsencoding = sys.getfilesystemencoding() fsencoding = sys.getfilesystemencoding()
if fsencoding is None: if fsencoding is None:
# This is very unlikely to happen, but if it does, just use UTF-8 # This is very unlikely to happen, but if it does, just use UTF-8
fsencoding = 'utf-8' fsencoding = "utf-8"
return fsencoding return fsencoding
def encode_local(s): def encode_local(s):
return s.encode(get_local_encoding(), default_local_errors) return s.encode(get_local_encoding(), default_local_errors)
def decode_local(bs): def decode_local(bs):
return bs.decode(get_local_encoding(), default_local_errors) return bs.decode(get_local_encoding(), default_local_errors)

hgext/fsmonitor/pywatchman/load.py

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals # no unicode literals
from __future__ import absolute_import, division, print_function
import ctypes
try: try:
from . import bser from . import bser
except ImportError: except ImportError:
from . import pybser as bser from . import pybser as bser
import ctypes
EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00" EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00"
def _read_bytes(fp, buf): def _read_bytes(fp, buf):
"""Read bytes from a file-like object """Read bytes from a file-like object
@param fp: File-like object that implements read(int) @param fp: File-like object that implements read(int)
if read_len < len(header): if read_len < len(header):
return None return None
total_len = bser.pdu_len(buf) total_len = bser.pdu_len(buf)
if total_len > len(buf): if total_len > len(buf):
ctypes.resize(buf, total_len) ctypes.resize(buf, total_len)
body = (ctypes.c_char * (total_len - len(header))).from_buffer( body = (ctypes.c_char * (total_len - len(header))).from_buffer(
buf, len(header)) buf, len(header)
)
read_len = _read_bytes(fp, body) read_len = _read_bytes(fp, body)
if read_len < len(body): if read_len < len(body):
raise RuntimeError('bser data ended early') raise RuntimeError("bser data ended early")
return bser.loads( return bser.loads(
(ctypes.c_char * total_len).from_buffer(buf, 0), (ctypes.c_char * total_len).from_buffer(buf, 0),
mutable, mutable,
value_encoding, value_encoding,
value_errors) value_errors,
)

hgext/fsmonitor/pywatchman/pybser.py

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals # no unicode literals
from __future__ import absolute_import, division, print_function
import binascii import binascii
import collections import collections
import ctypes import ctypes
import struct import struct
import sys import sys
from . import ( from . import compat
compat,
)
BSER_ARRAY = b'\x00' BSER_ARRAY = b"\x00"
BSER_OBJECT = b'\x01' BSER_OBJECT = b"\x01"
BSER_BYTESTRING = b'\x02' BSER_BYTESTRING = b"\x02"
BSER_INT8 = b'\x03' BSER_INT8 = b"\x03"
BSER_INT16 = b'\x04' BSER_INT16 = b"\x04"
BSER_INT32 = b'\x05' BSER_INT32 = b"\x05"
BSER_INT64 = b'\x06' BSER_INT64 = b"\x06"
BSER_REAL = b'\x07' BSER_REAL = b"\x07"
BSER_TRUE = b'\x08' BSER_TRUE = b"\x08"
BSER_FALSE = b'\x09' BSER_FALSE = b"\x09"
BSER_NULL = b'\x0a' BSER_NULL = b"\x0a"
BSER_TEMPLATE = b'\x0b' BSER_TEMPLATE = b"\x0b"
BSER_SKIP = b'\x0c' BSER_SKIP = b"\x0c"
BSER_UTF8STRING = b'\x0d' BSER_UTF8STRING = b"\x0d"
if compat.PYTHON3: if compat.PYTHON3:
STRING_TYPES = (str, bytes) STRING_TYPES = (str, bytes)
unicode = str unicode = str
def tobytes(i): def tobytes(i):
return str(i).encode('ascii') return str(i).encode("ascii")
long = int long = int
else: else:
STRING_TYPES = (unicode, str) STRING_TYPES = (unicode, str)
tobytes = bytes tobytes = bytes
# Leave room for the serialization header, which includes # Leave room for the serialization header, which includes
# our overall length. To make things simpler, we'll use an # our overall length. To make things simpler, we'll use an
# int32 for the header # int32 for the header
EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00" EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00"
EMPTY_HEADER_V2 = b"\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00" EMPTY_HEADER_V2 = b"\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00"
def _int_size(x): def _int_size(x):
"""Return the smallest size int that can store the value""" """Return the smallest size int that can store the value"""
if -0x80 <= x <= 0x7F: if -0x80 <= x <= 0x7F:
return 1 return 1
elif -0x8000 <= x <= 0x7FFF: elif -0x8000 <= x <= 0x7FFF:
return 2 return 2
elif -0x80000000 <= x <= 0x7FFFFFFF: elif -0x80000000 <= x <= 0x7FFFFFFF:
return 4 return 4
elif long(-0x8000000000000000) <= x <= long(0x7FFFFFFFFFFFFFFF): elif long(-0x8000000000000000) <= x <= long(0x7FFFFFFFFFFFFFFF):
return 8 return 8
else: else:
raise RuntimeError('Cannot represent value: ' + str(x)) raise RuntimeError("Cannot represent value: " + str(x))
def _buf_pos(buf, pos): def _buf_pos(buf, pos):
ret = buf[pos] ret = buf[pos]
# In Python 2, buf is a str array so buf[pos] is a string. In Python 3, buf # Normalize the return type to bytes
# is a bytes array and buf[pos] is an integer. if compat.PYTHON3 and not isinstance(ret, bytes):
if compat.PYTHON3:
ret = bytes((ret,)) ret = bytes((ret,))
return ret return ret
class _bser_buffer(object):
class _bser_buffer(object):
def __init__(self, version): def __init__(self, version):
self.bser_version = version self.bser_version = version
self.buf = ctypes.create_string_buffer(8192) self.buf = ctypes.create_string_buffer(8192)
if self.bser_version == 1: if self.bser_version == 1:
struct.pack_into(tobytes(len(EMPTY_HEADER)) + b's', self.buf, 0, struct.pack_into(
EMPTY_HEADER) tobytes(len(EMPTY_HEADER)) + b"s", self.buf, 0, EMPTY_HEADER
)
self.wpos = len(EMPTY_HEADER) self.wpos = len(EMPTY_HEADER)
else: else:
assert self.bser_version == 2 assert self.bser_version == 2
struct.pack_into(tobytes(len(EMPTY_HEADER_V2)) + b's', self.buf, 0, struct.pack_into(
EMPTY_HEADER_V2) tobytes(len(EMPTY_HEADER_V2)) + b"s",
self.buf,
0,
EMPTY_HEADER_V2,
)
self.wpos = len(EMPTY_HEADER_V2) self.wpos = len(EMPTY_HEADER_V2)
def ensure_size(self, size): def ensure_size(self, size):
while ctypes.sizeof(self.buf) - self.wpos < size: while ctypes.sizeof(self.buf) - self.wpos < size:
ctypes.resize(self.buf, ctypes.sizeof(self.buf) * 2) ctypes.resize(self.buf, ctypes.sizeof(self.buf) * 2)
def append_long(self, val): def append_long(self, val):
size = _int_size(val) size = _int_size(val)
to_write = size + 1 to_write = size + 1
self.ensure_size(to_write) self.ensure_size(to_write)
if size == 1: if size == 1:
struct.pack_into(b'=cb', self.buf, self.wpos, BSER_INT8, val) struct.pack_into(b"=cb", self.buf, self.wpos, BSER_INT8, val)
elif size == 2: elif size == 2:
struct.pack_into(b'=ch', self.buf, self.wpos, BSER_INT16, val) struct.pack_into(b"=ch", self.buf, self.wpos, BSER_INT16, val)
elif size == 4: elif size == 4:
struct.pack_into(b'=ci', self.buf, self.wpos, BSER_INT32, val) struct.pack_into(b"=ci", self.buf, self.wpos, BSER_INT32, val)
elif size == 8: elif size == 8:
struct.pack_into(b'=cq', self.buf, self.wpos, BSER_INT64, val) struct.pack_into(b"=cq", self.buf, self.wpos, BSER_INT64, val)
else: else:
raise RuntimeError('Cannot represent this long value') raise RuntimeError("Cannot represent this long value")
self.wpos += to_write self.wpos += to_write
def append_string(self, s): def append_string(self, s):
if isinstance(s, unicode): if isinstance(s, unicode):
s = s.encode('utf-8') s = s.encode("utf-8")
s_len = len(s) s_len = len(s)
size = _int_size(s_len) size = _int_size(s_len)
to_write = 2 + size + s_len to_write = 2 + size + s_len
self.ensure_size(to_write) self.ensure_size(to_write)
if size == 1: if size == 1:
struct.pack_into(b'=ccb' + tobytes(s_len) + b's', self.buf, struct.pack_into(
self.wpos, BSER_BYTESTRING, BSER_INT8, s_len, s) b"=ccb" + tobytes(s_len) + b"s",
self.buf,
self.wpos,
BSER_BYTESTRING,
BSER_INT8,
s_len,
s,
)
elif size == 2: elif size == 2:
struct.pack_into(b'=cch' + tobytes(s_len) + b's', self.buf, struct.pack_into(
self.wpos, BSER_BYTESTRING, BSER_INT16, s_len, s) b"=cch" + tobytes(s_len) + b"s",
self.buf,
self.wpos,
BSER_BYTESTRING,
BSER_INT16,
s_len,
s,
)
elif size == 4: elif size == 4:
struct.pack_into(b'=cci' + tobytes(s_len) + b's', self.buf, struct.pack_into(
self.wpos, BSER_BYTESTRING, BSER_INT32, s_len, s) b"=cci" + tobytes(s_len) + b"s",
self.buf,
self.wpos,
BSER_BYTESTRING,
BSER_INT32,
s_len,
s,
)
elif size == 8: elif size == 8:
struct.pack_into(b'=ccq' + tobytes(s_len) + b's', self.buf, struct.pack_into(
self.wpos, BSER_BYTESTRING, BSER_INT64, s_len, s) b"=ccq" + tobytes(s_len) + b"s",
self.buf,
self.wpos,
BSER_BYTESTRING,
BSER_INT64,
s_len,
s,
)
else: else:
raise RuntimeError('Cannot represent this string value') raise RuntimeError("Cannot represent this string value")
self.wpos += to_write self.wpos += to_write
def append_recursive(self, val): def append_recursive(self, val):
if isinstance(val, bool): if isinstance(val, bool):
needed = 1 needed = 1
self.ensure_size(needed) self.ensure_size(needed)
if val: if val:
to_encode = BSER_TRUE to_encode = BSER_TRUE
else: else:
to_encode = BSER_FALSE to_encode = BSER_FALSE
struct.pack_into(b'=c', self.buf, self.wpos, to_encode) struct.pack_into(b"=c", self.buf, self.wpos, to_encode)
self.wpos += needed self.wpos += needed
elif val is None: elif val is None:
needed = 1 needed = 1
self.ensure_size(needed) self.ensure_size(needed)
struct.pack_into(b'=c', self.buf, self.wpos, BSER_NULL) struct.pack_into(b"=c", self.buf, self.wpos, BSER_NULL)
self.wpos += needed self.wpos += needed
elif isinstance(val, (int, long)): elif isinstance(val, (int, long)):
self.append_long(val) self.append_long(val)
elif isinstance(val, STRING_TYPES): elif isinstance(val, STRING_TYPES):
self.append_string(val) self.append_string(val)
elif isinstance(val, float): elif isinstance(val, float):
needed = 9 needed = 9
self.ensure_size(needed) self.ensure_size(needed)
struct.pack_into(b'=cd', self.buf, self.wpos, BSER_REAL, val) struct.pack_into(b"=cd", self.buf, self.wpos, BSER_REAL, val)
self.wpos += needed self.wpos += needed
elif isinstance(val, collections.Mapping) and \ elif isinstance(val, collections.Mapping) and isinstance(
isinstance(val, collections.Sized): val, collections.Sized
):
val_len = len(val) val_len = len(val)
size = _int_size(val_len) size = _int_size(val_len)
needed = 2 + size needed = 2 + size
self.ensure_size(needed) self.ensure_size(needed)
if size == 1: if size == 1:
struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_OBJECT, struct.pack_into(
BSER_INT8, val_len) b"=ccb",
self.buf,
self.wpos,
BSER_OBJECT,
BSER_INT8,
val_len,
)
elif size == 2: elif size == 2:
struct.pack_into(b'=cch', self.buf, self.wpos, BSER_OBJECT, struct.pack_into(
BSER_INT16, val_len) b"=cch",
self.buf,
self.wpos,
BSER_OBJECT,
BSER_INT16,
val_len,
)
elif size == 4: elif size == 4:
struct.pack_into(b'=cci', self.buf, self.wpos, BSER_OBJECT, struct.pack_into(
BSER_INT32, val_len) b"=cci",
self.buf,
self.wpos,
BSER_OBJECT,
BSER_INT32,
val_len,
)
elif size == 8: elif size == 8:
struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_OBJECT, struct.pack_into(
BSER_INT64, val_len) b"=ccq",
self.buf,
self.wpos,
BSER_OBJECT,
BSER_INT64,
val_len,
)
else: else:
raise RuntimeError('Cannot represent this mapping value') raise RuntimeError("Cannot represent this mapping value")
self.wpos += needed self.wpos += needed
if compat.PYTHON3: if compat.PYTHON3:
iteritems = val.items() iteritems = val.items()
else: else:
iteritems = val.iteritems() iteritems = val.iteritems() # noqa: B301 Checked version above
for k, v in iteritems: for k, v in iteritems:
self.append_string(k) self.append_string(k)
self.append_recursive(v) self.append_recursive(v)
elif isinstance(val, collections.Iterable) and \ elif isinstance(val, collections.Iterable) and isinstance(
isinstance(val, collections.Sized): val, collections.Sized
):
val_len = len(val) val_len = len(val)
size = _int_size(val_len) size = _int_size(val_len)
needed = 2 + size needed = 2 + size
self.ensure_size(needed) self.ensure_size(needed)
if size == 1: if size == 1:
struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_ARRAY, struct.pack_into(
BSER_INT8, val_len) b"=ccb", self.buf, self.wpos, BSER_ARRAY, BSER_INT8, val_len
)
elif size == 2: elif size == 2:
struct.pack_into(b'=cch', self.buf, self.wpos, BSER_ARRAY, struct.pack_into(
BSER_INT16, val_len) b"=cch",
self.buf,
self.wpos,
BSER_ARRAY,
BSER_INT16,
val_len,
)
elif size == 4: elif size == 4:
struct.pack_into(b'=cci', self.buf, self.wpos, BSER_ARRAY, struct.pack_into(
BSER_INT32, val_len) b"=cci",
self.buf,
self.wpos,
BSER_ARRAY,
BSER_INT32,
val_len,
)
elif size == 8: elif size == 8:
struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_ARRAY, struct.pack_into(
BSER_INT64, val_len) b"=ccq",
self.buf,
self.wpos,
BSER_ARRAY,
BSER_INT64,
val_len,
)
else: else:
raise RuntimeError('Cannot represent this sequence value') raise RuntimeError("Cannot represent this sequence value")
self.wpos += needed self.wpos += needed
for v in val: for v in val:
self.append_recursive(v) self.append_recursive(v)
else: else:
raise RuntimeError('Cannot represent unknown value type') raise RuntimeError("Cannot represent unknown value type")
def dumps(obj, version=1, capabilities=0): def dumps(obj, version=1, capabilities=0):
bser_buf = _bser_buffer(version=version) bser_buf = _bser_buffer(version=version)
bser_buf.append_recursive(obj) bser_buf.append_recursive(obj)
# Now fill in the overall length # Now fill in the overall length
if version == 1: if version == 1:
obj_len = bser_buf.wpos - len(EMPTY_HEADER) obj_len = bser_buf.wpos - len(EMPTY_HEADER)
struct.pack_into(b'=i', bser_buf.buf, 3, obj_len) struct.pack_into(b"=i", bser_buf.buf, 3, obj_len)
else: else:
obj_len = bser_buf.wpos - len(EMPTY_HEADER_V2) obj_len = bser_buf.wpos - len(EMPTY_HEADER_V2)
struct.pack_into(b'=i', bser_buf.buf, 2, capabilities) struct.pack_into(b"=i", bser_buf.buf, 2, capabilities)
struct.pack_into(b'=i', bser_buf.buf, 7, obj_len) struct.pack_into(b"=i", bser_buf.buf, 7, obj_len)
return bser_buf.buf.raw[:bser_buf.wpos] return bser_buf.buf.raw[: bser_buf.wpos]
# This is a quack-alike with the bserObjectType in bser.c # This is a quack-alike with the bserObjectType in bser.c
# It provides by getattr accessors and getitem for both index # It provides by getattr accessors and getitem for both index
# and name. # and name.
class _BunserDict(object): class _BunserDict(object):
__slots__ = ('_keys', '_values') __slots__ = ("_keys", "_values")
def __init__(self, keys, values): def __init__(self, keys, values):
self._keys = keys self._keys = keys
self._values = values self._values = values
def __getattr__(self, name): def __getattr__(self, name):
return self.__getitem__(name) return self.__getitem__(name)
def __getitem__(self, key): def __getitem__(self, key):
if isinstance(key, (int, long)): if isinstance(key, (int, long)):
return self._values[key] return self._values[key]
elif key.startswith('st_'): elif key.startswith("st_"):
# hack^Wfeature to allow mercurial to use "st_size" to # hack^Wfeature to allow mercurial to use "st_size" to
# reference "size" # reference "size"
key = key[3:] key = key[3:]
try: try:
return self._values[self._keys.index(key)] return self._values[self._keys.index(key)]
except ValueError: except ValueError:
raise KeyError('_BunserDict has no key %s' % key) raise KeyError("_BunserDict has no key %s" % key)
def __len__(self): def __len__(self):
return len(self._keys) return len(self._keys)
class Bunser(object): class Bunser(object):
def __init__(self, mutable=True, value_encoding=None, value_errors=None): def __init__(self, mutable=True, value_encoding=None, value_errors=None):
self.mutable = mutable self.mutable = mutable
self.value_encoding = value_encoding self.value_encoding = value_encoding
if value_encoding is None: if value_encoding is None:
self.value_errors = None self.value_errors = None
elif value_errors is None: elif value_errors is None:
self.value_errors = 'strict' self.value_errors = "strict"
else: else:
self.value_errors = value_errors self.value_errors = value_errors
@staticmethod @staticmethod
def unser_int(buf, pos): def unser_int(buf, pos):
try: try:
int_type = _buf_pos(buf, pos) int_type = _buf_pos(buf, pos)
except IndexError: except IndexError:
raise ValueError('Invalid bser int encoding, pos out of range') raise ValueError("Invalid bser int encoding, pos out of range")
if int_type == BSER_INT8: if int_type == BSER_INT8:
needed = 2 needed = 2
fmt = b'=b' fmt = b"=b"
elif int_type == BSER_INT16: elif int_type == BSER_INT16:
needed = 3 needed = 3
fmt = b'=h' fmt = b"=h"
elif int_type == BSER_INT32: elif int_type == BSER_INT32:
needed = 5 needed = 5
fmt = b'=i' fmt = b"=i"
elif int_type == BSER_INT64: elif int_type == BSER_INT64:
needed = 9 needed = 9
fmt = b'=q' fmt = b"=q"
else: else:
raise ValueError('Invalid bser int encoding 0x%s' % raise ValueError(
binascii.hexlify(int_type).decode('ascii')) "Invalid bser int encoding 0x%s at position %s"
% (binascii.hexlify(int_type).decode("ascii"), pos)
)
int_val = struct.unpack_from(fmt, buf, pos + 1)[0] int_val = struct.unpack_from(fmt, buf, pos + 1)[0]
return (int_val, pos + needed) return (int_val, pos + needed)
def unser_utf8_string(self, buf, pos): def unser_utf8_string(self, buf, pos):
str_len, pos = self.unser_int(buf, pos + 1) str_len, pos = self.unser_int(buf, pos + 1)
str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0] str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0]
return (str_val.decode('utf-8'), pos + str_len) return (str_val.decode("utf-8"), pos + str_len)
def unser_bytestring(self, buf, pos): def unser_bytestring(self, buf, pos):
str_len, pos = self.unser_int(buf, pos + 1) str_len, pos = self.unser_int(buf, pos + 1)
str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0] str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0]
if self.value_encoding is not None: if self.value_encoding is not None:
str_val = str_val.decode(self.value_encoding, self.value_errors) str_val = str_val.decode(self.value_encoding, self.value_errors)
# str_len stays the same because that's the length in bytes # str_len stays the same because that's the length in bytes
return (str_val, pos + str_len) return (str_val, pos + str_len)
def unser_array(self, buf, pos): def unser_array(self, buf, pos):
arr_len, pos = self.unser_int(buf, pos + 1) arr_len, pos = self.unser_int(buf, pos + 1)
arr = [] arr = []
for i in range(arr_len): for _ in range(arr_len):
arr_item, pos = self.loads_recursive(buf, pos) arr_item, pos = self.loads_recursive(buf, pos)
arr.append(arr_item) arr.append(arr_item)
if not self.mutable: if not self.mutable:
arr = tuple(arr) arr = tuple(arr)
return arr, pos return arr, pos
def unser_object(self, buf, pos): def unser_object(self, buf, pos):
obj_len, pos = self.unser_int(buf, pos + 1) obj_len, pos = self.unser_int(buf, pos + 1)
if self.mutable: if self.mutable:
obj = {} obj = {}
else: else:
keys = [] keys = []
vals = [] vals = []
for i in range(obj_len): for _ in range(obj_len):
key, pos = self.unser_utf8_string(buf, pos) key, pos = self.unser_utf8_string(buf, pos)
val, pos = self.loads_recursive(buf, pos) val, pos = self.loads_recursive(buf, pos)
if self.mutable: if self.mutable:
obj[key] = val obj[key] = val
else: else:
keys.append(key) keys.append(key)
vals.append(val) vals.append(val)
if not self.mutable: if not self.mutable:
obj = _BunserDict(keys, vals) obj = _BunserDict(keys, vals)
return obj, pos return obj, pos
def unser_template(self, buf, pos): def unser_template(self, buf, pos):
val_type = _buf_pos(buf, pos + 1) val_type = _buf_pos(buf, pos + 1)
if val_type != BSER_ARRAY: if val_type != BSER_ARRAY:
raise RuntimeError('Expect ARRAY to follow TEMPLATE') raise RuntimeError("Expect ARRAY to follow TEMPLATE")
# force UTF-8 on keys # force UTF-8 on keys
keys_bunser = Bunser(mutable=self.mutable, value_encoding='utf-8') keys_bunser = Bunser(mutable=self.mutable, value_encoding="utf-8")
keys, pos = keys_bunser.unser_array(buf, pos + 1) keys, pos = keys_bunser.unser_array(buf, pos + 1)
nitems, pos = self.unser_int(buf, pos) nitems, pos = self.unser_int(buf, pos)
arr = [] arr = []
for i in range(nitems): for _ in range(nitems):
if self.mutable: if self.mutable:
obj = {} obj = {}
else: else:
vals = [] vals = []
for keyidx in range(len(keys)): for keyidx in range(len(keys)):
if _buf_pos(buf, pos) == BSER_SKIP: if _buf_pos(buf, pos) == BSER_SKIP:
pos += 1 pos += 1
if not self.mutable: if not self.mutable:
obj = _BunserDict(keys, vals) obj = _BunserDict(keys, vals)
arr.append(obj) arr.append(obj)
return arr, pos return arr, pos
def loads_recursive(self, buf, pos): def loads_recursive(self, buf, pos):
val_type = _buf_pos(buf, pos) val_type = _buf_pos(buf, pos)
if (val_type == BSER_INT8 or val_type == BSER_INT16 or if (
val_type == BSER_INT32 or val_type == BSER_INT64): val_type == BSER_INT8
or val_type == BSER_INT16
or val_type == BSER_INT32
or val_type == BSER_INT64
):
return self.unser_int(buf, pos) return self.unser_int(buf, pos)
elif val_type == BSER_REAL: elif val_type == BSER_REAL:
val = struct.unpack_from(b'=d', buf, pos + 1)[0] val = struct.unpack_from(b"=d", buf, pos + 1)[0]
return (val, pos + 9) return (val, pos + 9)
elif val_type == BSER_TRUE: elif val_type == BSER_TRUE:
return (True, pos + 1) return (True, pos + 1)
elif val_type == BSER_FALSE: elif val_type == BSER_FALSE:
return (False, pos + 1) return (False, pos + 1)
elif val_type == BSER_NULL: elif val_type == BSER_NULL:
return (None, pos + 1) return (None, pos + 1)
elif val_type == BSER_BYTESTRING: elif val_type == BSER_BYTESTRING:
return self.unser_bytestring(buf, pos) return self.unser_bytestring(buf, pos)
elif val_type == BSER_UTF8STRING: elif val_type == BSER_UTF8STRING:
return self.unser_utf8_string(buf, pos) return self.unser_utf8_string(buf, pos)
elif val_type == BSER_ARRAY: elif val_type == BSER_ARRAY:
return self.unser_array(buf, pos) return self.unser_array(buf, pos)
elif val_type == BSER_OBJECT: elif val_type == BSER_OBJECT:
return self.unser_object(buf, pos) return self.unser_object(buf, pos)
elif val_type == BSER_TEMPLATE: elif val_type == BSER_TEMPLATE:
return self.unser_template(buf, pos) return self.unser_template(buf, pos)
else: else:
raise ValueError('unhandled bser opcode 0x%s' % raise ValueError(
binascii.hexlify(val_type).decode('ascii')) "unhandled bser opcode 0x%s"
% binascii.hexlify(val_type).decode("ascii")
)
def _pdu_info_helper(buf): def _pdu_info_helper(buf):
bser_version = -1
if buf[0:2] == EMPTY_HEADER[0:2]: if buf[0:2] == EMPTY_HEADER[0:2]:
bser_version = 1 bser_version = 1
bser_capabilities = 0 bser_capabilities = 0
expected_len, pos2 = Bunser.unser_int(buf, 2) expected_len, pos2 = Bunser.unser_int(buf, 2)
elif buf[0:2] == EMPTY_HEADER_V2[0:2]: elif buf[0:2] == EMPTY_HEADER_V2[0:2]:
if len(buf) < 8: if len(buf) < 8:
raise ValueError('Invalid BSER header') raise ValueError("Invalid BSER header")
bser_version = 2 bser_version = 2
bser_capabilities = struct.unpack_from("I", buf, 2)[0] bser_capabilities = struct.unpack_from("I", buf, 2)[0]
expected_len, pos2 = Bunser.unser_int(buf, 6) expected_len, pos2 = Bunser.unser_int(buf, 6)
else: else:
raise ValueError('Invalid BSER header') raise ValueError("Invalid BSER header")
return bser_version, bser_capabilities, expected_len, pos2 return bser_version, bser_capabilities, expected_len, pos2
def pdu_info(buf): def pdu_info(buf):
info = _pdu_info_helper(buf) info = _pdu_info_helper(buf)
return info[0], info[1], info[2] + info[3] return info[0], info[1], info[2] + info[3]
@type value_errors: str @type value_errors: str
""" """
info = _pdu_info_helper(buf) info = _pdu_info_helper(buf)
expected_len = info[2] expected_len = info[2]
pos = info[3] pos = info[3]
if len(buf) != expected_len + pos: if len(buf) != expected_len + pos:
raise ValueError('bser data len != header len') raise ValueError(
"bser data len %d != header len %d" % (expected_len + pos, len(buf))
)
bunser = Bunser(mutable=mutable, value_encoding=value_encoding, bunser = Bunser(
value_errors=value_errors) mutable=mutable,
value_encoding=value_encoding,
value_errors=value_errors,
)
return bunser.loads_recursive(buf, pos)[0] return bunser.loads_recursive(buf, pos)[0]
def load(fp, mutable=True, value_encoding=None, value_errors=None): def load(fp, mutable=True, value_encoding=None, value_errors=None):
from . import load from . import load
return load.load(fp, mutable, value_encoding, value_errors) return load.load(fp, mutable, value_encoding, value_errors)

hgext/fsmonitor/watchmanclient.py

# watchmanclient.py - Watchman client for the fsmonitor extension # watchmanclient.py - Watchman client for the fsmonitor extension
# #
# Copyright 2013-2016 Facebook, Inc. # Copyright 2013-2016 Facebook, Inc.
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import getpass import getpass
from mercurial import util from mercurial import util
from mercurial.utils import procutil
from . import pywatchman from . import pywatchman
class Unavailable(Exception): class Unavailable(Exception):
def __init__(self, msg, warn=True, invalidate=False): def __init__(self, msg, warn=True, invalidate=False):
self.msg = msg self.msg = msg
self.warn = warn self.warn = warn
if self._watchmanclient is None: if self._watchmanclient is None:
self._firsttime = False self._firsttime = False
watchman_exe = self._ui.configpath( watchman_exe = self._ui.configpath(
b'fsmonitor', b'watchman_exe' b'fsmonitor', b'watchman_exe'
) )
self._watchmanclient = pywatchman.client( self._watchmanclient = pywatchman.client(
timeout=self._timeout, timeout=self._timeout,
useImmutableBser=True, useImmutableBser=True,
watchman_exe=watchman_exe, binpath=procutil.tonativestr(watchman_exe),
) )
return self._watchmanclient.query(*watchmanargs) return self._watchmanclient.query(*watchmanargs)
except pywatchman.CommandError as ex: except pywatchman.CommandError as ex:
if b'unable to resolve root' in ex.msg: if b'unable to resolve root' in ex.msg:
raise WatchmanNoRoot(self._root, ex.msg) raise WatchmanNoRoot(self._root, ex.msg)
raise Unavailable(ex.msg) raise Unavailable(ex.msg)
except pywatchman.WatchmanError as ex: except pywatchman.WatchmanError as ex:
raise Unavailable(str(ex)) raise Unavailable(str(ex))