This is an archive of the discontinued Mercurial Phabricator instance.

Status	Author	Revision
Closed	indygreg	D5760 remotefilelog: cast division result to an int
Closed	indygreg	D5759 tests: cast division result to int
Closed	indygreg	D5758 tests: various Python 3 ports for test-remotefilelog-datapack.py
Closed	indygreg	D5757 tests: use items() in test-remotefilelog-datapack.py
Closed	indygreg	D5756 tests: use bytes and %d formatting in test-remotefilelog-datapack.py

Diff 13593

tests/test-remotefilelog-datapack.py

	def getHash(self, content):			def getHash(self, content):
	return hashlib.sha1(content).digest()			return hashlib.sha1(content).digest()

	def getFakeHash(self):			def getFakeHash(self):
	return ''.join(chr(random.randint(0, 255)) for _ in range(20))			return ''.join(chr(random.randint(0, 255)) for _ in range(20))

	def createPack(self, revisions=None, packdir=None):			def createPack(self, revisions=None, packdir=None):
	if revisions is None:			if revisions is None:
	revisions = [("filename", self.getFakeHash(), nullid, "content")]			revisions = [(b"filename", self.getFakeHash(), nullid, b"content")]

	if packdir is None:			if packdir is None:
	packdir = self.makeTempDir()			packdir = self.makeTempDir()

	packer = datapack.mutabledatapack(uimod.ui(), packdir, version=2)			packer = datapack.mutabledatapack(uimod.ui(), packdir, version=2)

	for args in revisions:			for args in revisions:
	filename, node, base, content = args[0:4]			filename, node, base, content = args[0:4]
	# meta is optional			# meta is optional
	meta = None			meta = None
	if len(args) > 4:			if len(args) > 4:
	meta = args[4]			meta = args[4]
	packer.add(filename, node, base, content, metadata=meta)			packer.add(filename, node, base, content, metadata=meta)

	path = packer.close()			path = packer.close()
	return self.datapackreader(path)			return self.datapackreader(path)

	def _testAddSingle(self, content):			def _testAddSingle(self, content):
	"""Test putting a simple blob into a pack and reading it out.			"""Test putting a simple blob into a pack and reading it out.
	"""			"""
	filename = "foo"			filename = b"foo"
	node = self.getHash(content)			node = self.getHash(content)

	revisions = [(filename, node, nullid, content)]			revisions = [(filename, node, nullid, content)]
	pack = self.createPack(revisions)			pack = self.createPack(revisions)
	if self.paramsavailable:			if self.paramsavailable:
	self.assertEqual(pack.params.fanoutprefix,			self.assertEqual(pack.params.fanoutprefix,
	basepack.SMALLFANOUTPREFIX)			basepack.SMALLFANOUTPREFIX)

	chain = pack.getdeltachain(filename, node)			chain = pack.getdeltachain(filename, node)
	self.assertEqual(content, chain[0][4])			self.assertEqual(content, chain[0][4])

	def testAddSingle(self):			def testAddSingle(self):
	self._testAddSingle('')			self._testAddSingle(b'')

	def testAddSingleEmpty(self):			def testAddSingleEmpty(self):
	self._testAddSingle('abcdef')			self._testAddSingle(b'abcdef')

	def testAddMultiple(self):			def testAddMultiple(self):
	"""Test putting multiple unrelated blobs into a pack and reading them			"""Test putting multiple unrelated blobs into a pack and reading them
	out.			out.
	"""			"""
	revisions = []			revisions = []
	for i in range(10):			for i in range(10):
	filename = "foo%s" % i			filename = b"foo%d" % i
	content = "abcdef%s" % i			content = b"abcdef%d" % i
	node = self.getHash(content)			node = self.getHash(content)
	revisions.append((filename, node, self.getFakeHash(), content))			revisions.append((filename, node, self.getFakeHash(), content))

	pack = self.createPack(revisions)			pack = self.createPack(revisions)

	for filename, node, base, content in revisions:			for filename, node, base, content in revisions:
	entry = pack.getdelta(filename, node)			entry = pack.getdelta(filename, node)
	self.assertEqual((content, filename, base, {}), entry)			self.assertEqual((content, filename, base, {}), entry)

	chain = pack.getdeltachain(filename, node)			chain = pack.getdeltachain(filename, node)
	self.assertEqual(content, chain[0][4])			self.assertEqual(content, chain[0][4])

	def testAddDeltas(self):			def testAddDeltas(self):
	"""Test putting multiple delta blobs into a pack and read the chain.			"""Test putting multiple delta blobs into a pack and read the chain.
	"""			"""
	revisions = []			revisions = []
	filename = "foo"			filename = b"foo"
	lastnode = nullid			lastnode = nullid
	for i in range(10):			for i in range(10):
	content = "abcdef%s" % i			content = b"abcdef%d" % i
	node = self.getHash(content)			node = self.getHash(content)
	revisions.append((filename, node, lastnode, content))			revisions.append((filename, node, lastnode, content))
	lastnode = node			lastnode = node

	pack = self.createPack(revisions)			pack = self.createPack(revisions)

	entry = pack.getdelta(filename, revisions[0][1])			entry = pack.getdelta(filename, revisions[0][1])
	realvalue = (revisions[0][3], filename, revisions[0][2], {})			realvalue = (revisions[0][3], filename, revisions[0][2], {})
	self.assertEqual(entry, realvalue)			self.assertEqual(entry, realvalue)

	# Test that the chain for the final entry has all the others			# Test that the chain for the final entry has all the others
	chain = pack.getdeltachain(filename, node)			chain = pack.getdeltachain(filename, node)
	for i in range(10):			for i in range(10):
	content = "abcdef%s" % i			content = b"abcdef%d" % i
	self.assertEqual(content, chain[-i - 1][4])			self.assertEqual(content, chain[-i - 1][4])

	def testPackMany(self):			def testPackMany(self):
	"""Pack many related and unrelated objects.			"""Pack many related and unrelated objects.
	"""			"""
	# Build a random pack file			# Build a random pack file
	revisions = []			revisions = []
	blobs = {}			blobs = {}
	random.seed(0)			random.seed(0)
	for i in range(100):			for i in range(100):
	filename = "filename-%s" % i			filename = b"filename-%d" % i
	filerevs = []			filerevs = []
	for j in range(random.randint(1, 100)):			for j in range(random.randint(1, 100)):
	content = "content-%s" % j			content = b"content-%d" % j
	node = self.getHash(content)			node = self.getHash(content)
	lastnode = nullid			lastnode = nullid
	if len(filerevs) > 0:			if len(filerevs) > 0:
	lastnode = filerevs[random.randint(0, len(filerevs) - 1)]			lastnode = filerevs[random.randint(0, len(filerevs) - 1)]
	filerevs.append(node)			filerevs.append(node)
	blobs[(filename, node, lastnode)] = content			blobs[(filename, node, lastnode)] = content
	revisions.append((filename, node, lastnode, content))			revisions.append((filename, node, lastnode, content))

	pack = self.createPack(revisions)			pack = self.createPack(revisions)

	# Verify the pack contents			# Verify the pack contents
	for (filename, node, lastnode), content in sorted(blobs.iteritems()):			for (filename, node, lastnode), content in sorted(blobs.iteritems()):
	chain = pack.getdeltachain(filename, node)			chain = pack.getdeltachain(filename, node)
	for entry in chain:			for entry in chain:
	expectedcontent = blobs[(entry[0], entry[1], entry[3])]			expectedcontent = blobs[(entry[0], entry[1], entry[3])]
	self.assertEqual(entry[4], expectedcontent)			self.assertEqual(entry[4], expectedcontent)

	def testPackMetadata(self):			def testPackMetadata(self):
	revisions = []			revisions = []
	for i in range(100):			for i in range(100):
	filename = '%s.txt' % i			filename = b'%d.txt' % i
	content = 'put-something-here \n' * i			content = b'put-something-here \n' * i
	node = self.getHash(content)			node = self.getHash(content)
	meta = {constants.METAKEYFLAG: i ** 4,			meta = {constants.METAKEYFLAG: i ** 4,
	constants.METAKEYSIZE: len(content),			constants.METAKEYSIZE: len(content),
	'Z': 'random_string',			b'Z': b'random_string',
	'_': '\0' * i}			b'_': b'\0' * i}
	revisions.append((filename, node, nullid, content, meta))			revisions.append((filename, node, nullid, content, meta))
	pack = self.createPack(revisions)			pack = self.createPack(revisions)
	for name, node, x, content, origmeta in revisions:			for name, node, x, content, origmeta in revisions:
	parsedmeta = pack.getmeta(name, node)			parsedmeta = pack.getmeta(name, node)
	# flag == 0 should be optimized out			# flag == 0 should be optimized out
	if origmeta[constants.METAKEYFLAG] == 0:			if origmeta[constants.METAKEYFLAG] == 0:
	del origmeta[constants.METAKEYFLAG]			del origmeta[constants.METAKEYFLAG]
	self.assertEqual(parsedmeta, origmeta)			self.assertEqual(parsedmeta, origmeta)

	def testGetMissing(self):			def testGetMissing(self):
	"""Test the getmissing() api.			"""Test the getmissing() api.
	"""			"""
	revisions = []			revisions = []
	filename = "foo"			filename = b"foo"
	lastnode = nullid			lastnode = nullid
	for i in range(10):			for i in range(10):
	content = "abcdef%s" % i			content = b"abcdef%d" % i
	node = self.getHash(content)			node = self.getHash(content)
	revisions.append((filename, node, lastnode, content))			revisions.append((filename, node, lastnode, content))
	lastnode = node			lastnode = node

	pack = self.createPack(revisions)			pack = self.createPack(revisions)

	missing = pack.getmissing([("foo", revisions[0][1])])			missing = pack.getmissing([(b"foo", revisions[0][1])])
	self.assertFalse(missing)			self.assertFalse(missing)

	missing = pack.getmissing([("foo", revisions[0][1]),			missing = pack.getmissing([(b"foo", revisions[0][1]),
	("foo", revisions[1][1])])			(b"foo", revisions[1][1])])
	self.assertFalse(missing)			self.assertFalse(missing)

	fakenode = self.getFakeHash()			fakenode = self.getFakeHash()
	missing = pack.getmissing([("foo", revisions[0][1]), ("foo", fakenode)])			missing = pack.getmissing([(b"foo", revisions[0][1]), (b"foo", fakenode)])
	self.assertEqual(missing, [("foo", fakenode)])			self.assertEqual(missing, [(b"foo", fakenode)])

	def testAddThrows(self):			def testAddThrows(self):
	pack = self.createPack()			pack = self.createPack()

	try:			try:
	pack.add('filename', nullid, 'contents')			pack.add(b'filename', nullid, b'contents')
	self.assertTrue(False, "datapack.add should throw")			self.assertTrue(False, "datapack.add should throw")
	except RuntimeError:			except RuntimeError:
	pass			pass

	def testBadVersionThrows(self):			def testBadVersionThrows(self):
	pack = self.createPack()			pack = self.createPack()
	path = pack.path + '.datapack'			path = pack.path + b'.datapack'
	with open(path) as f:			with open(path) as f:
	raw = f.read()			raw = f.read()
	raw = struct.pack('!B', 255) + raw[1:]			raw = struct.pack('!B', 255) + raw[1:]
	os.chmod(path, os.stat(path).st_mode \| stat.S_IWRITE)			os.chmod(path, os.stat(path).st_mode \| stat.S_IWRITE)
	with open(path, 'w+') as f:			with open(path, 'w+') as f:
	f.write(raw)			f.write(raw)

	try:			try:
	pack = self.datapackreader(pack.path)			pack = self.datapackreader(pack.path)
	self.assertTrue(False, "bad version number should have thrown")			self.assertTrue(False, "bad version number should have thrown")
	except RuntimeError:			except RuntimeError:
	pass			pass

	def testMissingDeltabase(self):			def testMissingDeltabase(self):
	fakenode = self.getFakeHash()			fakenode = self.getFakeHash()
	revisions = [("filename", fakenode, self.getFakeHash(), "content")]			revisions = [(b"filename", fakenode, self.getFakeHash(), b"content")]
	pack = self.createPack(revisions)			pack = self.createPack(revisions)
	chain = pack.getdeltachain("filename", fakenode)			chain = pack.getdeltachain(b"filename", fakenode)
	self.assertEqual(len(chain), 1)			self.assertEqual(len(chain), 1)

	def testLargePack(self):			def testLargePack(self):
	"""Test creating and reading from a large pack with over X entries.			"""Test creating and reading from a large pack with over X entries.
	This causes it to use a 2^16 fanout table instead."""			This causes it to use a 2^16 fanout table instead."""
	revisions = []			revisions = []
	blobs = {}			blobs = {}
	total = basepack.SMALLFANOUTCUTOFF + 1			total = basepack.SMALLFANOUTCUTOFF + 1
	for i in pycompat.xrange(total):			for i in pycompat.xrange(total):
	filename = "filename-%s" % i			filename = b"filename-%d" % i
	content = filename			content = filename
	node = self.getHash(content)			node = self.getHash(content)
	blobs[(filename, node)] = content			blobs[(filename, node)] = content
	revisions.append((filename, node, nullid, content))			revisions.append((filename, node, nullid, content))

	pack = self.createPack(revisions)			pack = self.createPack(revisions)
	if self.paramsavailable:			if self.paramsavailable:
	self.assertEqual(pack.params.fanoutprefix,			self.assertEqual(pack.params.fanoutprefix,
	packdir = self.makeTempDir()			packdir = self.makeTempDir()
	deltachains = []			deltachains = []

	numpacks = 10			numpacks = 10
	revisionsperpack = 100			revisionsperpack = 100

	for i in range(numpacks):			for i in range(numpacks):
	chain = []			chain = []
	revision = (str(i), self.getFakeHash(), nullid, "content")			revision = (b'%d' % i, self.getFakeHash(), nullid, b"content")

	for _ in range(revisionsperpack):			for _ in range(revisionsperpack):
	chain.append(revision)			chain.append(revision)
	revision = (			revision = (
	str(i),			b'%d' % i,
	self.getFakeHash(),			self.getFakeHash(),
	revision[1],			revision[1],
	self.getFakeHash()			self.getFakeHash()
	)			)

	self.createPack(chain, packdir)			self.createPack(chain, packdir)
	deltachains.append(chain)			deltachains.append(chain)

	1000,			1000,
	10000,			10000,
	100000,			100000,
	1000000,			1000000,
	]			]
	for packsize in packsizes:			for packsize in packsizes:
	revisions = []			revisions = []
	for i in pycompat.xrange(packsize):			for i in pycompat.xrange(packsize):
	filename = "filename-%s" % i			filename = b"filename-%d" % i
	content = "content-%s" % i			content = b"content-%d" % i
	node = self.getHash(content)			node = self.getHash(content)
	revisions.append((filename, node, nullid, content))			revisions.append((filename, node, nullid, content))

	path = self.createPack(revisions).path			path = self.createPack(revisions).path

	# Perf of large multi-get			# Perf of large multi-get
	import gc			import gc
	gc.disable()			gc.disable()
	pack = self.datapackreader(path)			pack = self.datapackreader(path)
	for lookupsize in lookupsizes:			for lookupsize in lookupsizes:
	if lookupsize > packsize:			if lookupsize > packsize:
	continue			continue
	random.shuffle(revisions)			random.shuffle(revisions)
	findnodes = [(rev[0], rev[1]) for rev in revisions]			findnodes = [(rev[0], rev[1]) for rev in revisions]

	start = time.time()			start = time.time()
	pack.getmissing(findnodes[:lookupsize])			pack.getmissing(findnodes[:lookupsize])
	elapsed = time.time() - start			elapsed = time.time() - start
	print ("%s pack %s lookups = %0.04f" %			print ("%s pack %d lookups = %0.04f" %
	(('%s' % packsize).rjust(7),			(('%d' % packsize).rjust(7),
	('%s' % lookupsize).rjust(7),			('%d' % lookupsize).rjust(7),
	elapsed))			elapsed))

	print("")			print("")
	gc.enable()			gc.enable()

	# The perf test is meant to produce output, so we always fail the test			# The perf test is meant to produce output, so we always fail the test
	# so the user sees the output.			# so the user sees the output.
	raise RuntimeError("perf test always fails")			raise RuntimeError("perf test always fails")

Diff	ID	Description	Created	Lint	Unit
Base		Base
Diff 1	13593		Jan 30 2019, 4:37 PM	★	★
Diff 2	13600	rHG5aa3fe343042cb71de4d5e5b1a9ce47cada0d842	Jan 30 2019, 4:21 PM	★	★

This is an archive of the discontinued Mercurial Phabricator instance.

tests: use bytes and %d formatting in test-remotefilelog-datapack.pyClosedPublic

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 13593

tests/test-remotefilelog-datapack.py

tests: use bytes and %d formatting in test-remotefilelog-datapack.py
ClosedPublic

Revision Contents
Changeset List