diff --git a/remotefilelog/fileserverclient.py b/remotefilelog/fileserverclient.py --- a/remotefilelog/fileserverclient.py +++ b/remotefilelog/fileserverclient.py @@ -9,6 +9,7 @@ import hashlib, os, time, io, struct import itertools +import subprocess from mercurial.i18n import _ from mercurial.node import hex, bin, nullid @@ -117,16 +118,56 @@ gets and sets by communicating with an external process that has the cache-specific implementation. """ - def __init__(self): + def __init__(self, repo): + self.repo = repo self.pipeo = self.pipei = self.pipee = None self.subprocess = None self.connected = False + # default Linux pipe buffer size for kernels >=2.6.11 + self.pipei_bufsize = 65536 if os.name != 'nt' else 4096 + + def _defaultconnect(self, cachecommand): + self.pipei, self.pipeo, self.pipee, self.subprocess = \ + util.popen4(cachecommand) + + def _windowsconnect(self, cachecommand): + import msvcrt + from _subprocess import CreatePipe + # By default, Windows has a pipe buffer size of 4K. This is the + # reason why we saw deadlocks in _getfiles on Windows but not on POSIX + # To circumvent this, we ask the system to create a pipe with + # a large enough buffer. + # Unfortunately, we can't do it cleanly with subprocess.Popen API: + # it's bufsize argument is related to the in-process (setvbuf) buffer, + # not the OS pipe buffer itself. + # This method converts between three types of file specifiers: + # - native Windows HANDLEs, returned by CreatePipe + # - C Runtime file descriptors, obtained by open_osfhandle + # - Python file-like objects, obtained by os.fdopen + handles = CreatePipe(None, self.pipei_bufsize) + # subprocess.py cannot take real HANDLEs, needs fds + rfd, wfd = [msvcrt.open_osfhandle(h, 0) for h in handles] + self.subprocess = subprocess.Popen(cachecommand, newlines=False, + close_fds=False, env=None, + stdin=rfd, # manual pipe + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.pipeo = self.subprocess.stdout + self.pipee = self.subprocess.stderr + # turn file descriptor into a file-like object + # bufsize=-1 means "use default C Runtime buffer for an open fd" + self.pipei = os.fdopen(wfd, 'wb', bufsize=-1) def connect(self, cachecommand): if self.pipeo: raise error.Abort(_("cache connection already open")) - self.pipei, self.pipeo, self.pipee, self.subprocess = \ - util.popen4(cachecommand) + custompipei = self.repo.ui.configbool('remotefilelog', + 'windows.custompipei', True) + if os.name == 'nt' and custompipei: + # to ensure we have pipe buffer of the same size as on Linux + self._windowsconnect(cachecommand) + else: + self._defaultconnect(cachecommand) self.connected = True def close(self): @@ -266,7 +307,7 @@ self.debugoutput = ui.configbool("remotefilelog", "debug") - self.remotecache = cacheconnection() + self.remotecache = cacheconnection(repo) self.connpool = connectionpool.connectionpool(repo) def setstore(self, datastore, historystore, writedata, writehistory): diff --git a/tests/test-check-config-hg.t b/tests/test-check-config-hg.t --- a/tests/test-check-config-hg.t +++ b/tests/test-check-config-hg.t @@ -70,6 +70,7 @@ undocumented: remotefilelog.shallowtrees (bool) undocumented: remotefilelog.validatecache (str) ['on'] undocumented: remotefilelog.validatecachelog (str) + undocumented: remotefilelog.windows.custompipei (bool) [True] undocumented: simplecache.cachedir (str) undocumented: simplecache.caches (list) [['local']] undocumented: simplecache.evictionpercent (int) [50]