cstore/datastore.h
150	Seems this line should be deleted?
cstore/key.h
16 ↗	(On Diff #1387)	Why is this required?

cstore/key.h
16 ↗	(On Diff #1387)	Ooops. That's strange that I missed that. It is not required now, it was needed for my previous implementation of python data stores and it got here by accident.

cstore/datapackstore.h
23	alphabetical
cstore/datastore.h
110	I'm guessing freedeltachain doesn't actually free the delta_chain_t memory itself, since we're not passing in a pointer. So we probably need to free that up manually to avoid a memory leak.

Diff	ID	Description	Created	Lint	Unit
Base		Base
Diff 1	1387		Aug 29 2017, 9:51 AM	★	★
Diff 2	1431	Removed unnecessary import and removed a line	Aug 30 2017, 7:07 AM	★	★
Diff 3	1437	Rebased onto stable and added necessary includes so that build is successful on…	Aug 30 2017, 10:36 AM	★	★
Diff 4	1438	Edited datastore.h file descriptions to match the standard	Aug 30 2017, 10:56 AM	★	★
Diff 5	1439	Updated testing plan	Aug 30 2017, 10:59 AM	★	★
Diff 6	1446	DeltaChain costructor no longer accepts a pointer to the delta_chain_t. It…	Aug 30 2017, 2:26 PM	★	★
Diff 7	1494	rFBHGX8cacf9669a0af58bf17398bdbb457eeed282c7a2	Aug 30 2017, 2:37 PM	★	★

Diff 1494

cstore/datapackstore.h

	#ifndef FBHGEXT_DATAPACKSTORE_H			#ifndef FBHGEXT_DATAPACKSTORE_H
	#define FBHGEXT_DATAPACKSTORE_H			#define FBHGEXT_DATAPACKSTORE_H

	extern "C" {			extern "C" {
	#include "cdatapack/cdatapack.h"			#include "cdatapack/cdatapack.h"
	}			}

	#include <ctime>			#include <ctime>
				#include <memory>
	#include <string>			#include <string>
	#include <unordered_set>			#include <unordered_set>
	#include <vector>			#include <vector>

	#include "clib/portability/portability.h"			#include "cstore/datastore.h"
				durhamUnsubmitted Done alphabetical durham: alphabetical
	#include "cstore/key.h"			#include "cstore/key.h"
				#include "clib/portability/portability.h"

	const clock_t PACK_REFRESH_RATE = 0.1 * CLOCKS_PER_SEC;			const clock_t PACK_REFRESH_RATE = 0.1 * CLOCKS_PER_SEC;

	class DeltaChainIterator {
	private:
	size_t _index;
	protected:
	std::vector<delta_chain_t> _chains;
	DeltaChainIterator() :
	_index(0) {}
	virtual delta_chain_t getNextChain(const Key& /key/) {
	return COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND };
	}
	public:
	DeltaChainIterator(delta_chain_t chain) :
	_index(0) {
	_chains.push_back(chain);
	}

	virtual ~DeltaChainIterator() {
	for(std::vector<delta_chain_t>::iterator it = _chains.begin();
	it != _chains.end();
	it++) {
	freedeltachain(*it);
	}
	}

	delta_chain_link_t *next() {
	delta_chain_t *chain = &_chains.back();
	if (_index >= chain->links_count) {
	// If we're not at the end, and we have a callback to fetch more, do the
	// fetch.
	bool refreshed = false;
	if (chain->links_count > 0) {
	delta_chain_link_t *result = &chain->delta_chain_links[_index - 1];

	const uint8_t *deltabasenode = result->deltabase_node;
	if (memcmp(deltabasenode, NULLID, BIN_NODE_SIZE) != 0) {
	Key key(result->filename, result->filename_sz,
	(const char*)deltabasenode, BIN_NODE_SIZE);

	delta_chain_t newChain = this->getNextChain(key);
	if (newChain.code == GET_DELTA_CHAIN_OK) {
	// Do not free the old chain, since the iterator consumer may
	// still be holding references to it.
	_chains.push_back(newChain);
	chain = &_chains.back();
	_index = 0;
	refreshed = true;
	} else {
	freedeltachain(newChain);
	}
	}
	}

	if (!refreshed) {
	return NULL;
	}
	}

	delta_chain_link_t *result = &chain->delta_chain_links[_index];
	_index++;

	return result;
	}
	};

	class DatapackStore;			class DatapackStore;
	class DatapackStoreKeyIterator : public KeyIterator {			class DatapackStoreKeyIterator : public KeyIterator {
	private:			private:
	DatapackStore &_store;			DatapackStore &_store;
	KeyIterator &_missing;			KeyIterator &_missing;

	public:			public:
	DatapackStoreKeyIterator(DatapackStore &store, KeyIterator &missing) :			DatapackStoreKeyIterator(DatapackStore &store, KeyIterator &missing) :
	_store(store),			_store(store),
	_missing(missing) {}			_missing(missing) {}

	Key *next() override;			Key *next() override;
	};			};

	/* Manages access to a directory of datapack files. */			/* Manages access to a directory of datapack files. */
	class DatapackStore {			class DatapackStore : public DataStore {
	private:			private:
	std::string _path;			std::string _path;
	clock_t _lastRefresh;			clock_t _lastRefresh;

	std::unordered_set<std::string> _packPaths;			std::unordered_set<std::string> _packPaths;

	datapack_handle_t *addPack(const std::string &path);			datapack_handle_t *addPack(const std::string &path);

	std::vector<datapack_handle_t*> refresh();			std::vector<datapack_handle_t*> refresh();
	public:			public:
	std::vector<datapack_handle_t*> _packs;			std::vector<datapack_handle_t*> _packs;

	DatapackStore(const std::string &path);			DatapackStore(const std::string &path);

	~DatapackStore();			~DatapackStore();

	DeltaChainIterator getDeltaChain(const Key &key);			DeltaChainIterator getDeltaChain(const Key &key);

	DatapackStoreKeyIterator getMissing(KeyIterator &missing);			std::shared_ptr<KeyIterator> getMissing(KeyIterator &missing);

	delta_chain_t getDeltaChainRaw(const Key &key);			std::shared_ptr<DeltaChain> getDeltaChainRaw(const Key &key);

	bool contains(const Key &key);			bool contains(const Key &key);

	void markForRefresh();			void markForRefresh();
	};			};

	#endif // FBHGEXT_DATAPACKSTORE_H			#endif // FBHGEXT_DATAPACKSTORE_H

cstore/datapackstore.cpp

	for(std::vector<datapack_handle_t*>::iterator it = _packs.begin();			for(std::vector<datapack_handle_t*>::iterator it = _packs.begin();
	it != _packs.end();			it != _packs.end();
	it++) {			it++) {
	close_datapack(*it);			close_datapack(*it);
	}			}
	}			}

	DeltaChainIterator DatapackStore::getDeltaChain(const Key &key) {			DeltaChainIterator DatapackStore::getDeltaChain(const Key &key) {
	delta_chain_t chain = this->getDeltaChainRaw(key);			std::shared_ptr<DeltaChain> chain = this->getDeltaChainRaw(key);
	if (chain.code == GET_DELTA_CHAIN_OK) {			if (chain->code() == GET_DELTA_CHAIN_OK) {
	return DeltaChainIterator(chain);			return DeltaChainIterator(chain);
	}			}

	freedeltachain(chain);

	throw MissingKeyError("unable to find delta chain");			throw MissingKeyError("unable to find delta chain");
	}			}

	delta_chain_t DatapackStore::getDeltaChainRaw(const Key &key) {			std::shared_ptr<DeltaChain> DatapackStore::getDeltaChainRaw(const Key &key) {
	for(std::vector<datapack_handle_t*>::iterator it = _packs.begin();			for(std::vector<datapack_handle_t*>::iterator it = _packs.begin();
	it != _packs.end();			it != _packs.end();
	it++) {			it++) {
	datapack_handle_t pack = it;			datapack_handle_t pack = it;

	delta_chain_t chain = getdeltachain(pack, (const uint8_t *) key.node);			delta_chain_t chain = getdeltachain(pack, (const uint8_t *) key.node);
	if (chain.code == GET_DELTA_CHAIN_OOM) {			if (chain.code == GET_DELTA_CHAIN_OOM) {
	throw std::runtime_error("out of memory");			throw std::runtime_error("out of memory");
	} else if (chain.code == GET_DELTA_CHAIN_NOT_FOUND) {			} else if (chain.code == GET_DELTA_CHAIN_NOT_FOUND) {
	freedeltachain(chain);			freedeltachain(chain);
	continue;			continue;
	} else if (chain.code != GET_DELTA_CHAIN_OK) {			} else if (chain.code != GET_DELTA_CHAIN_OK) {
	freedeltachain(chain);			freedeltachain(chain);
	continue;			continue;
	}			}

	return chain;			// Pass ownership of chain to DeltaChain
				return std::make_shared<DeltaChain>(chain);
	}			}

	// Check if there are new packs available			// Check if there are new packs available
	std::vector<datapack_handle_t*> refreshed = refresh();			std::vector<datapack_handle_t*> refreshed = refresh();
	for(std::vector<datapack_handle_t*>::iterator it = refreshed.begin();			for(std::vector<datapack_handle_t*>::iterator it = refreshed.begin();
	it != refreshed.end();			it != refreshed.end();
	it++) {			it++) {
	datapack_handle_t pack = it;			datapack_handle_t pack = it;

	delta_chain_t chain = getdeltachain(pack, (const uint8_t *) key.node);			delta_chain_t chain = getdeltachain(pack, (const uint8_t *) key.node);
	if (chain.code == GET_DELTA_CHAIN_OOM) {			if (chain.code == GET_DELTA_CHAIN_OOM) {
	throw std::runtime_error("out of memory");			throw std::runtime_error("out of memory");
	} else if (chain.code == GET_DELTA_CHAIN_NOT_FOUND) {			} else if (chain.code == GET_DELTA_CHAIN_NOT_FOUND) {
	freedeltachain(chain);			freedeltachain(chain);
	continue;			continue;
	} else if (chain.code != GET_DELTA_CHAIN_OK) {			} else if (chain.code != GET_DELTA_CHAIN_OK) {
	freedeltachain(chain);			freedeltachain(chain);
	continue;			continue;
	}			}

	return chain;			// Pass ownership of chain to DeltaChain
				return std::make_shared<DeltaChain>(chain);
	}			}

	return COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND };			return std::make_shared<DeltaChain>(GET_DELTA_CHAIN_NOT_FOUND);
	}			}

	Key *DatapackStoreKeyIterator::next() {			Key *DatapackStoreKeyIterator::next() {
	Key *key;			Key *key;
	while ((key = _missing.next()) != NULL) {			while ((key = _missing.next()) != NULL) {
	if (!_store.contains(*key)) {			if (!_store.contains(*key)) {
	return key;			return key;
	}			}
	if (find(pack, (uint8_t*)key.node, &packindex)) {			if (find(pack, (uint8_t*)key.node, &packindex)) {
	return true;			return true;
	}			}
	}			}

	return false;			return false;
	}			}

	DatapackStoreKeyIterator DatapackStore::getMissing(KeyIterator &missing) {			std::shared_ptr<KeyIterator> DatapackStore::getMissing(KeyIterator &missing) {
	return DatapackStoreKeyIterator(*this, missing);			return std::make_shared<DatapackStoreKeyIterator>(*this, missing);
	}			}

	std::vector<datapack_handle_t*> DatapackStore::refresh() {			std::vector<datapack_handle_t*> DatapackStore::refresh() {
	clock_t now = clock();			clock_t now = clock();

	std::vector<datapack_handle_t*> newPacks;			std::vector<datapack_handle_t*> newPacks;
	if (now - _lastRefresh > PACK_REFRESH_RATE) {			if (now - _lastRefresh > PACK_REFRESH_RATE) {
	std::vector<std::string> availablePacks = getAvailablePackFiles(_path);			std::vector<std::string> availablePacks = getAvailablePackFiles(_path);

cstore/datastore.h

This file was added.

				// Copyright (c) 2004-present, Facebook, Inc.
				// All Rights Reserved.
				//
				// This software may be used and distributed according to the terms of the
				// GNU General Public License version 2 or any later version.

				// datastore.h - c++ declarations for a data store
				// no-check-code

				#ifndef FBHGEXT_DATASTORE_H
				#define FBHGEXT_DATASTORE_H

				extern "C" {
				#include "cdatapack/cdatapack.h"
				}

				#include <memory>
				#include <vector>

				#include "cstore/key.h"
				#include "clib/portability/portability.h"

				class DeltaChainLink {
				private:
				const char _filename, _deltabasefilename;
				const uint8_t _node, _deltabasenode, *_delta;
				uint16_t _filenamesz;
				uint64_t _deltasz;

				public:
				DeltaChainLink(delta_chain_link_t *link) {
				if (link) {
				_filename = link->filename;
				_deltabasefilename = link->filename;
				_node = link->node;
				_deltabasenode = link->deltabase_node;
				_delta = link->delta;
				_filenamesz = link->filename_sz;
				_deltasz = link->delta_sz;
				} else {
				_filename = NULL;
				_deltabasefilename = NULL;
				_node = NULL;
				_deltabasenode = NULL;
				_delta = NULL;
				_filenamesz = 0;
				_deltasz = 0;
				}
				}

				const char* filename() {
				return _filename;
				}

				const char* deltabasefilename() {
				return _deltabasefilename;
				}

				const uint8_t* node() {
				return _node;
				}

				const uint8_t* deltabasenode() {
				return _deltabasenode;
				}

				const uint8_t* delta() {
				return _delta;
				}

				uint16_t filenamesz() {
				return _filenamesz;
				}

				uint64_t deltasz() {
				return _deltasz;
				}

				bool isdone() {
				return (_filename == NULL);
				}
				};

				/*
				* This class takes ownership of a delta chain
				*/
				class DeltaChain {
				private:
				//C DeltaChain
				delta_chain_t _chain;

				public:
				//The constructor does a shallow copy of the delta chain and since the
				//ownership is taken by this class it is responsible for memory management
				DeltaChain(delta_chain_t chain) : _chain(chain) {}

				DeltaChain(get_delta_chain_code_t error) {
				_chain = COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND };
				}

				~DeltaChain() {
				freedeltachain(_chain);
				}

				const DeltaChainLink getlink(const size_t idx) {
				return DeltaChainLink(&(_chain.delta_chain_links[idx]));
				}

				size_t linkcount() {
				return _chain.links_count;
				durhamUnsubmitted Done I'm guessing freedeltachain doesn't actually free the delta_chain_t memory itself, since we're not passing in a pointer. So we probably need to free that up manually to avoid a memory leak. durham: I'm guessing freedeltachain doesn't actually free the delta_chain_t memory itself, since we're…
				}

				get_delta_chain_code_t code() {
				return _chain.code;
				}

				};

				class DeltaChainIterator {
				private:
				size_t _index;
				protected:
				std::vector< std::shared_ptr<DeltaChain> > _chains;
				DeltaChainIterator() :
				_index(0) {}
				virtual std::shared_ptr<DeltaChain> getNextChain(const Key &key) {
				return std::make_shared<DeltaChain>(GET_DELTA_CHAIN_NOT_FOUND);
				}
				public:
				DeltaChainIterator(std::shared_ptr<DeltaChain> chain) :
				_index(0) {
				_chains.push_back(chain);
				}

				DeltaChainLink next() {
				std::shared_ptr<DeltaChain> chain = _chains.back();

				if (_index >= chain->linkcount()) {
				// If we're not at the end, and we have a callback to fetch more, do the
				// fetch.
				bool refreshed = false;
				if (chain->linkcount() > 0) {
				DeltaChainLink result = chain->getlink(_index - 1);

				const uint8_t *deltabasenode = result.deltabasenode();
				if (memcmp(deltabasenode, NULLID, BIN_NODE_SIZE) != 0) {
				Key key(result.filename(), result.filenamesz(),
				(const char*)deltabasenode, BIN_NODE_SIZE);

				std::shared_ptr<DeltaChain> newChain = this->getNextChain(key);
				durhamUnsubmitted Done Seems this line should be deleted? durham: Seems this line should be deleted?
				if (newChain->code() == GET_DELTA_CHAIN_OK) {
				// Do not free the old chain, since the iterator consumer may
				// still be holding references to it.
				_chains.push_back(newChain);
				chain = _chains.back();
				_index = 0;
				refreshed = true;
				}
				}
				}

				if (!refreshed) {
				return DeltaChainLink(NULL);
				}
				}

				DeltaChainLink result = chain->getlink(_index);
				_index++;

				return result;
				}
				};

				class DataStore {
				protected:
				DataStore() {}

				public:
				virtual ~DataStore() {}

				virtual DeltaChainIterator getDeltaChain(const Key &key) = 0;

				virtual std::shared_ptr<DeltaChain> getDeltaChainRaw(const Key &key) = 0;

				virtual std::shared_ptr<KeyIterator> getMissing(KeyIterator &missing) = 0;

				virtual bool contains(const Key &key) = 0;

				virtual void markForRefresh() = 0;
				};

				#endif // FBHGEXT_DATASTORE_H

cstore/py-datapackstore.h

	#include <memory>			#include <memory>
	#include <string>			#include <string>

	extern "C" {			extern "C" {
	#include "cdatapack/cdatapack.h"			#include "cdatapack/cdatapack.h"
	}			}

	#include "cstore/datapackstore.h"			#include "cstore/datapackstore.h"
				#include "cstore/datastore.h"
	#include "cstore/key.h"			#include "cstore/key.h"
	#include "cstore/py-structs.h"			#include "cstore/py-structs.h"
	#include "cstore/pythonutil.h"			#include "cstore/pythonutil.h"
	#include "cstore/uniondatapackstore.h"			#include "cstore/uniondatapackstore.h"

	// --------- DatapackStore Implementation ---------			// --------- DatapackStore Implementation ---------

	/*			/*
	}			}

	Key key(name, namelen, node, nodelen);			Key key(name, namelen, node, nodelen);

	DeltaChainIterator chain = self->datapackstore.getDeltaChain(key);			DeltaChainIterator chain = self->datapackstore.getDeltaChain(key);

	PythonObj resultChain = PyList_New(0);			PythonObj resultChain = PyList_New(0);

	delta_chain_link_t *link;
	size_t index = 0;			size_t index = 0;
	while ((link = chain.next()) != NULL) {			for (DeltaChainLink link = chain.next(); !link.isdone(); link = chain.next()) {
	PythonObj name = PyString_FromStringAndSize(link->filename, link->filename_sz);			PythonObj name = PyString_FromStringAndSize(link.filename(), link.filenamesz());
	PythonObj retnode = PyString_FromStringAndSize((const char *) link->node, NODE_SZ);			PythonObj retnode = PyString_FromStringAndSize((const char *) link.node(), NODE_SZ);
	PythonObj deltabasenode = PyString_FromStringAndSize(			PythonObj deltabasenode = PyString_FromStringAndSize(
	(const char *) link->deltabase_node, NODE_SZ);			(const char *) link.deltabasenode(), NODE_SZ);
	PythonObj delta = PyString_FromStringAndSize(			PythonObj delta = PyString_FromStringAndSize(
	(const char *) link->delta, (Py_ssize_t) link->delta_sz);			(const char *) link.delta(), (Py_ssize_t) link.deltasz());

	PythonObj tuple = PyTuple_Pack(5, (PyObject)name, (PyObject)retnode,			PythonObj tuple = PyTuple_Pack(5, (PyObject)name, (PyObject)retnode,
	(PyObject)name, (PyObject)deltabasenode,			(PyObject)name, (PyObject)deltabasenode,
	(PyObject*)delta);			(PyObject*)delta);

	if (PyList_Append((PyObject*)resultChain, tuple.returnval())) {			if (PyList_Append((PyObject*)resultChain, tuple.returnval())) {
	return NULL;			return NULL;
	}			}

	static PyObject datapackstore_getmissing(py_datapackstore self, PyObject *keys) {			static PyObject datapackstore_getmissing(py_datapackstore self, PyObject *keys) {
	try {			try {
	PythonObj result = PyList_New(0);			PythonObj result = PyList_New(0);

	PythonObj inputIterator = PyObject_GetIter(keys);			PythonObj inputIterator = PyObject_GetIter(keys);
	PythonKeyIterator keysIter(inputIterator);			PythonKeyIterator keysIter(inputIterator);

	DatapackStoreKeyIterator missingIter = self->datapackstore.getMissing(keysIter);			std::shared_ptr<KeyIterator> missingIter = self->datapackstore.getMissing(keysIter);

	Key *key;			Key *key;
	while ((key = missingIter.next()) != NULL) {			while ((key = missingIter->next()) != NULL) {
	PythonObj missingKey = Py_BuildValue("(s#s#)", key->name.c_str(), key->name.size(),			PythonObj missingKey = Py_BuildValue("(s#s#)", key->name.c_str(), key->name.size(),
	key->node, 20);			key->node, 20);
	if (PyList_Append(result, (PyObject*)missingKey)) {			if (PyList_Append(result, (PyObject*)missingKey)) {
	return NULL;			return NULL;
	}			}
	}			}

	return result.returnval();			return result.returnval();
	*/			*/
	static int uniondatapackstore_init(py_uniondatapackstore self, PyObject args) {			static int uniondatapackstore_init(py_uniondatapackstore self, PyObject args) {
	PyObject *storeList;			PyObject *storeList;
	if (!PyArg_ParseTuple(args, "O", &storeList)) {			if (!PyArg_ParseTuple(args, "O", &storeList)) {
	return -1;			return -1;
	}			}

	try {			try {
	std::vector<DatapackStore*> stores;			std::vector<DataStore*> stores;
	std::vector<PythonObj> pySubStores;			std::vector<PythonObj> pySubStores;

	PyObject *item;			PyObject *item;
	PythonObj inputIterator = PyObject_GetIter(storeList);			PythonObj inputIterator = PyObject_GetIter(storeList);
	while ((item = PyIter_Next(inputIterator)) != NULL) {			while ((item = PyIter_Next(inputIterator)) != NULL) {
	// Record the substore references, so:			// Record the substore references, so:
	// A) We can decref them in case of an error.			// A) We can decref them in case of an error.
	// B) They don't get GC'd while the uniondatapackstore holds on to them.			// B) They don't get GC'd while the uniondatapackstore holds on to them.
	}			}

	Key key(name, namelen, node, nodelen);			Key key(name, namelen, node, nodelen);

	UnionDeltaChainIterator chain = self->uniondatapackstore->getDeltaChain(key);			UnionDeltaChainIterator chain = self->uniondatapackstore->getDeltaChain(key);

	PythonObj resultChain = PyList_New(0);			PythonObj resultChain = PyList_New(0);

	delta_chain_link_t *link;			for (DeltaChainLink link = chain.next(); !link.isdone(); link = chain.next()) {
	while ((link = chain.next()) != NULL) {			PythonObj name = PyString_FromStringAndSize(link.filename(), link.filenamesz());
	PythonObj name = PyString_FromStringAndSize(link->filename, link->filename_sz);			PythonObj retnode = PyString_FromStringAndSize((const char *) link.node(), NODE_SZ);
	PythonObj retnode = PyString_FromStringAndSize((const char *) link->node, NODE_SZ);
	PythonObj deltabasenode = PyString_FromStringAndSize(			PythonObj deltabasenode = PyString_FromStringAndSize(
	(const char *) link->deltabase_node, NODE_SZ);			(const char *) link.deltabasenode(), NODE_SZ);
	PythonObj delta = PyString_FromStringAndSize(			PythonObj delta = PyString_FromStringAndSize(
	(const char *) link->delta, (Py_ssize_t) link->delta_sz);			(const char *) link.delta(), (Py_ssize_t) link.deltasz());

	PythonObj tuple = PyTuple_Pack(5, (PyObject)name, (PyObject)retnode,			PythonObj tuple = PyTuple_Pack(5, (PyObject)name, (PyObject)retnode,
	(PyObject)name, (PyObject)deltabasenode,			(PyObject)name, (PyObject)deltabasenode,
	(PyObject*)delta);			(PyObject*)delta);

	if (PyList_Append((PyObject*)resultChain, tuple.returnval())) {			if (PyList_Append((PyObject*)resultChain, tuple.returnval())) {
	return NULL;			return NULL;
	}			}

cstore/uniondatapackstore.h


	Key *next() override;			Key *next() override;
	};			};

	class UnionDeltaChainIterator: public DeltaChainIterator {			class UnionDeltaChainIterator: public DeltaChainIterator {
	private:			private:
	UnionDatapackStore &_store;			UnionDatapackStore &_store;
	protected:			protected:
	delta_chain_t getNextChain(const Key &key) override;			std::shared_ptr<DeltaChain> getNextChain(const Key &key) override;

	public:			public:
	UnionDeltaChainIterator(UnionDatapackStore &store, const Key &key) :			UnionDeltaChainIterator(UnionDatapackStore &store, const Key &key) :
	DeltaChainIterator(),			DeltaChainIterator(),
	_store(store) {			_store(store) {
	_chains.push_back(this->getNextChain(key));			_chains.push_back(this->getNextChain(key));
	}			}
	};			};

	class UnionDatapackStore : public Store {			class UnionDatapackStore : public Store {
	public:			public:
	std::vector<DatapackStore*> _stores;			std::vector<DataStore*> _stores;

	UnionDatapackStore(std::vector<DatapackStore*> stores);			UnionDatapackStore(std::vector<DataStore*> stores);

	~UnionDatapackStore() override;			~UnionDatapackStore() override;

	ConstantStringRef get(const Key &key) override;			ConstantStringRef get(const Key &key) override;

	UnionDeltaChainIterator getDeltaChain(const Key &key);			UnionDeltaChainIterator getDeltaChain(const Key &key);

	bool contains(const Key &key);			bool contains(const Key &key);

	UnionDatapackStoreKeyIterator getMissing(KeyIterator &missing);			UnionDatapackStoreKeyIterator getMissing(KeyIterator &missing);

	void markForRefresh();			void markForRefresh();
	};			};

	#endif // FBHGEXT_CSTORE_UNIONDATAPACKSTORE_H			#endif // FBHGEXT_CSTORE_UNIONDATAPACKSTORE_H

cstore/uniondatapackstore.cpp


	#include <algorithm>			#include <algorithm>
	#include <memory>			#include <memory>

	extern "C" {			extern "C" {
	#include "cstore/mpatch.h"			#include "cstore/mpatch.h"
	}			}

	UnionDatapackStore::UnionDatapackStore(std::vector<DatapackStore*> stores) :			UnionDatapackStore::UnionDatapackStore(std::vector<DataStore*> stores) :
	_stores(stores) {			_stores(stores) {
	}			}

	UnionDatapackStore::~UnionDatapackStore() {			UnionDatapackStore::~UnionDatapackStore() {
	// TODO: we should manage the substore lifetimes here, but because they are			// TODO: we should manage the substore lifetimes here, but because they are
	// also controlled by Python, we need to let python handle it and manage the			// also controlled by Python, we need to let python handle it and manage the
	// refcount in the py_uniondatapackstore type.			// refcount in the py_uniondatapackstore type.
	}			}

	mpatch_flist* getNextLink(void* container, ssize_t index) {			mpatch_flist* getNextLink(void* container, ssize_t index) {
	std::vector<delta_chain_link_t> links = (std::vector<delta_chain_link_t>)container;			std::vector<DeltaChainLink> links = (std::vector<DeltaChainLink>)container;

	if (index < 0 \|\| (size_t)index >= links->size()) {			if (index < 0 \|\| (size_t)index >= links->size()) {
	return NULL;			return NULL;
	}			}

	delta_chain_link_t *link = links->at(index);			DeltaChainLink link = links->at(index);

	struct mpatch_flist *res;			struct mpatch_flist *res;
	if ((mpatch_decode((const char*)link->delta,			if ((mpatch_decode((const char*)link.delta(),
	(ssize_t)link->delta_sz, &res)) < 0) {			(ssize_t)link.deltasz(), &res)) < 0) {
	throw std::logic_error("invalid patch during patch application");			throw std::logic_error("invalid patch during patch application");
	}			}

	return res;			return res;
	}			}

	ConstantStringRef UnionDatapackStore::get(const Key &key) {			ConstantStringRef UnionDatapackStore::get(const Key &key) {
	UnionDeltaChainIterator chain = this->getDeltaChain(key);			UnionDeltaChainIterator chain = this->getDeltaChain(key);

	std::vector<delta_chain_link_t*> links;			std::vector<DeltaChainLink> links;

	delta_chain_link_t *link;			for (DeltaChainLink link = chain.next(); !link.isdone(); link = chain.next()) {
	while ((link = chain.next()) != NULL) {
	links.push_back(link);			links.push_back(link);
	}			}

	delta_chain_link_t *fulltextLink = links.back();			DeltaChainLink fulltextLink = links.back();
	links.pop_back();			links.pop_back();

	// Short circuit and just return the full text if it's one long			// Short circuit and just return the full text if it's one long
	if (links.size() == 0) {			if (links.size() == 0) {
	return ConstantStringRef((const char*)fulltextLink->delta,			return ConstantStringRef((const char*)fulltextLink.delta(),
	(size_t)fulltextLink->delta_sz);			(size_t)fulltextLink.deltasz());
	}			}

	std::reverse(links.begin(), links.end());			std::reverse(links.begin(), links.end());

	mpatch_flist *patch = mpatch_fold(&links, getNextLink, 0, links.size());			mpatch_flist *patch = mpatch_fold(&links, getNextLink, 0, links.size());
	if (!patch) { /* error already set or memory error */			if (!patch) { /* error already set or memory error */
	throw std::logic_error("mpatch failed to fold patches");			throw std::logic_error("mpatch failed to fold patches");
	}			}

	ssize_t outlen = mpatch_calcsize((ssize_t)fulltextLink->delta_sz, patch);			ssize_t outlen = mpatch_calcsize((ssize_t)fulltextLink.deltasz(), patch);
	if (outlen < 0) {			if (outlen < 0) {
	mpatch_lfree(patch);			mpatch_lfree(patch);
	throw std::logic_error("mpatch failed to calculate size");			throw std::logic_error("mpatch failed to calculate size");
	}			}

	auto result = std::make_shared<std::string>(outlen, '\0');			auto result = std::make_shared<std::string>(outlen, '\0');
	if (mpatch_apply(&(result)[0], (const char)fulltextLink->delta,			if (mpatch_apply(&(result)[0], (const char)fulltextLink.delta(),
	(ssize_t)fulltextLink->delta_sz, patch) < 0) {			(ssize_t)fulltextLink.deltasz(), patch) < 0) {
	mpatch_lfree(patch);			mpatch_lfree(patch);
	throw std::logic_error("mpatch failed to apply patches");			throw std::logic_error("mpatch failed to apply patches");
	}			}

	mpatch_lfree(patch);			mpatch_lfree(patch);
	return ConstantStringRef(result);			return ConstantStringRef(result);
	}			}

	delta_chain_t UnionDeltaChainIterator::getNextChain(const Key &key) {			std::shared_ptr<DeltaChain> UnionDeltaChainIterator::getNextChain(const Key &key) {
	for(std::vector<DatapackStore*>::iterator it = _store._stores.begin();			for(std::vector<DataStore*>::iterator it = _store._stores.begin();
	it != _store._stores.end();			it != _store._stores.end();
	it++) {			it++) {
	DatapackStore substore = it;			DataStore substore = it;
	delta_chain_t chain = substore->getDeltaChainRaw(key);			std::shared_ptr<DeltaChain> chain = substore->getDeltaChainRaw(key);
	if (chain.code == GET_DELTA_CHAIN_OK) {
				if (chain->code() == GET_DELTA_CHAIN_OK) {
	return chain;			return chain;
	}			}
	freedeltachain(chain);
	}			}

	throw MissingKeyError("unable to find delta chain");			throw MissingKeyError("unable to find delta chain");
	}			}

	UnionDeltaChainIterator UnionDatapackStore::getDeltaChain(const Key &key) {			UnionDeltaChainIterator UnionDatapackStore::getDeltaChain(const Key &key) {
	return UnionDeltaChainIterator(*this, key);			return UnionDeltaChainIterator(*this, key);
	}			}

	Key *UnionDatapackStoreKeyIterator::next() {			Key* UnionDatapackStoreKeyIterator::next() {
	Key *key;			Key* key;
	while ((key = _missing.next()) != NULL) {			while ((key = _missing.next()) != NULL) {
	if (!_store.contains(*key)) {			if (!_store.contains(*key)) {
	return key;			return key;
	}			}
	}			}

	return NULL;			return NULL;
	}			}

	bool UnionDatapackStore::contains(const Key &key) {			bool UnionDatapackStore::contains(const Key &key) {
	for(std::vector<DatapackStore*>::iterator it = _stores.begin();			for(std::vector<DataStore*>::iterator it = _stores.begin();
	it != _stores.end();			it != _stores.end();
	it++) {			it++) {
	DatapackStore substore = it;			DataStore substore = it;
	if (substore->contains(key)) {			if (substore->contains(key)) {
	return true;			return true;
	}			}
	}			}
	return false;			return false;
	}			}

	UnionDatapackStoreKeyIterator UnionDatapackStore::getMissing(KeyIterator &missing) {			UnionDatapackStoreKeyIterator UnionDatapackStore::getMissing(KeyIterator &missing) {
	return UnionDatapackStoreKeyIterator(*this, missing);			return UnionDatapackStoreKeyIterator(*this, missing);
	}			}

	void UnionDatapackStore::markForRefresh() {			void UnionDatapackStore::markForRefresh() {
	for(std::vector<DatapackStore*>::iterator it = _stores.begin();			for(std::vector<DataStore*>::iterator it = _stores.begin();
	it != _stores.end();			it != _stores.end();
	it++) {			it++) {
	DatapackStore substore = it;			DataStore substore = it;
	substore->markForRefresh();			substore->markForRefresh();
	}			}
	}			}

tests/test-check-code-hg.t

	Skipping clib/convert.h it has no-che?k-code (glob)			Skipping clib/convert.h it has no-che?k-code (glob)
	Skipping clib/null_test.c it has no-che?k-code (glob)			Skipping clib/null_test.c it has no-che?k-code (glob)
	Skipping clib/portability/inet.h it has no-che?k-code (glob)			Skipping clib/portability/inet.h it has no-che?k-code (glob)
	Skipping clib/portability/portability.h it has no-che?k-code (glob)			Skipping clib/portability/portability.h it has no-che?k-code (glob)
	Skipping clib/portability/unistd.h it has no-che?k-code (glob)			Skipping clib/portability/unistd.h it has no-che?k-code (glob)
	Skipping clib/sha1.h it has no-che?k-code (glob)			Skipping clib/sha1.h it has no-che?k-code (glob)
	Skipping cstore/datapackstore.cpp it has no-che?k-code (glob)			Skipping cstore/datapackstore.cpp it has no-che?k-code (glob)
	Skipping cstore/datapackstore.h it has no-che?k-code (glob)			Skipping cstore/datapackstore.h it has no-che?k-code (glob)
				Skipping cstore/datastore.h it has no-che?k-code (glob)
	Skipping cstore/key.h it has no-che?k-code (glob)			Skipping cstore/key.h it has no-che?k-code (glob)
	Skipping cstore/match.h it has no-che?k-code (glob)			Skipping cstore/match.h it has no-che?k-code (glob)
	Skipping cstore/py-cdatapack.h it has no-che?k-code (glob)			Skipping cstore/py-cdatapack.h it has no-che?k-code (glob)
	Skipping cstore/py-cstore.cpp it has no-che?k-code (glob)			Skipping cstore/py-cstore.cpp it has no-che?k-code (glob)
	Skipping cstore/py-datapackstore.h it has no-che?k-code (glob)			Skipping cstore/py-datapackstore.h it has no-che?k-code (glob)
	Skipping cstore/py-structs.h it has no-che?k-code (glob)			Skipping cstore/py-structs.h it has no-che?k-code (glob)
	Skipping cstore/py-treemanifest.h it has no-che?k-code (glob)			Skipping cstore/py-treemanifest.h it has no-che?k-code (glob)
	Skipping cstore/pythonutil.cpp it has no-che?k-code (glob)			Skipping cstore/pythonutil.cpp it has no-che?k-code (glob)

		Path
M		cstore/datapackstore.h (74 lines)
M		cstore/datapackstore.cpp (21 lines)
A	M	cstore/datastore.h (192 lines)
M		cstore/py-datapackstore.h (29 lines)
M		cstore/uniondatapackstore.h (6 lines)
M		cstore/uniondatapackstore.cpp (51 lines)
M		tests/test-check-code-hg.t (1 line)

This is an archive of the discontinued Mercurial Phabricator instance.

cstore: generic uniondatapackstore
ClosedPublic

Details

Diff Detail

Event Timeline

Revision Contents
Changeset List

Diff 1494

cstore/datapackstore.h

cstore/datapackstore.cpp

cstore/datastore.h

cstore/py-datapackstore.h

cstore/uniondatapackstore.h

cstore/uniondatapackstore.cpp

tests/test-check-code-hg.t

This is an archive of the discontinued Mercurial Phabricator instance.

cstore: generic uniondatapackstoreClosedPublic

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 1494

cstore/datapackstore.h

cstore/datapackstore.cpp

cstore/datastore.h

cstore/py-datapackstore.h

cstore/uniondatapackstore.h

cstore/uniondatapackstore.cpp

tests/test-check-code-hg.t

cstore: generic uniondatapackstore
ClosedPublic

Revision Contents
Changeset List