diff --git a/hgext3rd/extmmap.pyx b/hgext3rd/extmmap.pyx new file mode 100644 --- /dev/null +++ b/hgext3rd/extmmap.pyx @@ -0,0 +1,70 @@ +# extmmap.pyx - Simple mmap implementation that does not keep a fd open +# +# Copyright 2017 Facebook, Inc. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from cpython.bytes cimport PyBytes_FromStringAndSize +from libc.errno cimport errno +from posix cimport mman, stat + +import mmap as pymmap +import os + +cdef extern from "sys/mman.h": + cdef char *MAP_FAILED + +cdef _raise_oserror(message=''): + if message: + message += ': ' + message += os.strerror(errno) + return OSError(errno, message) + +cdef class mmap: + cdef char *ptr + cdef size_t len + + def __cinit__(self, int fd, size_t length, int access=pymmap.ACCESS_READ): + cdef stat.struct_stat st + # Only support a subset of options for now + if access != pymmap.ACCESS_READ: + raise RuntimeError('access %s is unsupported' % access) + + # If length is 0, read size from file + if length == 0: + r = stat.fstat(fd, &st) + if r != 0: + _raise_oserror('fstat failed') + length = st.st_size + + self.ptr = mman.mmap(NULL, length, mman.PROT_READ, + mman.MAP_SHARED, fd, 0) + if self.ptr == MAP_FAILED: + _raise_oserror('mmap failed') + + self.len = length + + # This method should be called "unmap". But it's to be compatible with + # the Python API. + cpdef close(self): + if self.ptr == NULL: + return + + r = mman.munmap(self.ptr, self.len) + if r != 0: + _raise_oserror('munmap failed') + self.ptr = NULL + + def __getslice__(self, Py_ssize_t i, Py_ssize_t j): + if self.ptr == NULL: + raise RuntimeError('mmap closed') + if j > self.len: + j = self.len + return PyBytes_FromStringAndSize(self.ptr + i, j - i) + + def __len__(self): + return self.len + + def __dealloc__(self): + self.close() diff --git a/remotefilelog/basepack.py b/remotefilelog/basepack.py --- a/remotefilelog/basepack.py +++ b/remotefilelog/basepack.py @@ -9,6 +9,14 @@ from . import shallowutil +mmapfunc = mmap.mmap +try: + from hgext3rd import extmmap +except ImportError: + pass +else: + mmapfunc = extmmap.mmap + osutil = policy.importmod(r'osutil') # The pack version supported by this implementation. This will need to be @@ -310,8 +318,9 @@ if self.VERSION == 0: return self.indexsize else: - nodecount = struct.unpack_from('!Q', self._index, - self.params.indexstart - 8)[0] + offset = self.params.indexstart - 8 + nodecount = struct.unpack_from('!Q', + self._index[offset:offset + 8])[0] return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH def freememory(self): @@ -328,10 +337,10 @@ # TODO: use an opener/vfs to access these paths with open(self.indexpath, PACKOPENMODE) as indexfp: # memory-map the file, size 0 means whole file - self._index = mmap.mmap(indexfp.fileno(), 0, - access=mmap.ACCESS_READ) + self._index = mmapfunc(indexfp.fileno(), 0, + access=mmap.ACCESS_READ) with open(self.packpath, PACKOPENMODE) as datafp: - self._data = mmap.mmap(datafp.fileno(), 0, access=mmap.ACCESS_READ) + self._data = mmapfunc(datafp.fileno(), 0, access=mmap.ACCESS_READ) self._pagedin = 0 return True diff --git a/remotefilelog/datapack.py b/remotefilelog/datapack.py --- a/remotefilelog/datapack.py +++ b/remotefilelog/datapack.py @@ -277,7 +277,7 @@ if self.VERSION == 1: # <4 byte len> + - metalen = struct.unpack_from('!I', data, offset)[0] + metalen = struct.unpack('!I', data[offset:offset + 4])[0] offset += 4 + metalen yield (filename, node, deltabase, uncompressedlen) diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -296,6 +296,14 @@ ] + cflags), ), ], + 'extmmap': [ + Extension('hgext3rd.extmmap', + sources=['hgext3rd/extmmap.pyx'], + extra_compile_args=filter(None, [ + STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC, + ]), + ), + ], 'linelog' : [ Extension('linelog', sources=['linelog/pyext/linelog.pyx'], @@ -354,6 +362,7 @@ cythonmodules = ['linelog'] else: cythonmodules = [ + 'extmmap', 'linelog', 'patchrmdir', 'traceprof', diff --git a/tests/test-remotefilelog-datapack.py b/tests/test-remotefilelog-datapack.py --- a/tests/test-remotefilelog-datapack.py +++ b/tests/test-remotefilelog-datapack.py @@ -38,6 +38,7 @@ class datapacktestsbase(object): def __init__(self, datapackreader, paramsavailable): self.datapackreader = datapackreader + self.iscdatapack = not paramsavailable self.paramsavailable = paramsavailable def setUp(self): @@ -275,11 +276,11 @@ def testPacksCache(self): """Test that we remember the most recent packs while fetching the delta chain.""" - packdir = self.makeTempDir() deltachains = [] - numpacks = 10 + # Ensures that we are not keeping everything in the cache. + numpacks = datapackstore.DEFAULTCACHESIZE * 2 revisionsperpack = 100 for i in range(numpacks): @@ -298,11 +299,7 @@ self.createPack(chain, packdir) deltachains.append(chain) - class testdatapackstore(datapackstore): - # Ensures that we are not keeping everything in the cache. - DEFAULTCACHESIZE = numpacks / 2 - - store = testdatapackstore(mercurial.ui.ui(), packdir) + store = datapackstore(mercurial.ui.ui(), packdir, self.iscdatapack) random.shuffle(deltachains) for randomchain in deltachains: