diff --git a/cstore/datapackstore.cpp b/cstore/datapackstore.cpp --- a/cstore/datapackstore.cpp +++ b/cstore/datapackstore.cpp @@ -102,7 +102,7 @@ DeltaChainIterator DatapackStore::getDeltaChain(const Key &key) { std::shared_ptr chain = this->getDeltaChainRaw(key); - if (chain->code() == GET_DELTA_CHAIN_OK) { + if (chain->status() == GET_DELTA_CHAIN_OK) { return DeltaChainIterator(chain); } throw MissingKeyError("unable to find delta chain"); @@ -125,8 +125,8 @@ continue; } - // Pass ownership of chain to DeltaChain - return std::make_shared(chain); + // Pass ownership of chain to CDeltaChain + return std::make_shared(chain); } // Check if there are new packs available @@ -147,11 +147,11 @@ continue; } - // Pass ownership of chain to DeltaChain - return std::make_shared(chain); + // Pass ownership of chain to CDeltaChain + return std::make_shared(chain); } - return std::make_shared(GET_DELTA_CHAIN_NOT_FOUND); + return std::make_shared(GET_DELTA_CHAIN_NOT_FOUND); } Key *DatapackStoreKeyIterator::next() { diff --git a/cstore/datastore.h b/cstore/datastore.h --- a/cstore/datastore.h +++ b/cstore/datastore.h @@ -10,166 +10,10 @@ #ifndef FBHGEXT_DATASTORE_H #define FBHGEXT_DATASTORE_H -extern "C" { -#include "cdatapack/cdatapack.h" -} - #include -#include - -#include "cstore/key.h" -#include "clib/portability/portability.h" - -class DeltaChainLink { - private: - const char *_filename, *_deltabasefilename; - const uint8_t *_node, *_deltabasenode, *_delta; - uint16_t _filenamesz; - uint64_t _deltasz; - - public: - DeltaChainLink(delta_chain_link_t *link) { - if (link) { - _filename = link->filename; - _deltabasefilename = link->filename; - _node = link->node; - _deltabasenode = link->deltabase_node; - _delta = link->delta; - _filenamesz = link->filename_sz; - _deltasz = link->delta_sz; - } else { - _filename = NULL; - _deltabasefilename = NULL; - _node = NULL; - _deltabasenode = NULL; - _delta = NULL; - _filenamesz = 0; - _deltasz = 0; - } - } - - const char* filename() { - return _filename; - } - - const char* deltabasefilename() { - return _deltabasefilename; - } - - const uint8_t* node() { - return _node; - } - - const uint8_t* deltabasenode() { - return _deltabasenode; - } - - const uint8_t* delta() { - return _delta; - } - - uint16_t filenamesz() { - return _filenamesz; - } - - uint64_t deltasz() { - return _deltasz; - } - - bool isdone() { - return (_filename == NULL); - } -}; - -/* - * This class takes ownership of a delta chain - */ -class DeltaChain { - private: - //C DeltaChain - delta_chain_t _chain; - public: - //The constructor does a shallow copy of the delta chain and since the - //ownership is taken by this class it is responsible for memory management - DeltaChain(delta_chain_t chain) : _chain(chain) {} - - DeltaChain(get_delta_chain_code_t error) { - _chain = COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND }; - } - - ~DeltaChain() { - freedeltachain(_chain); - } - - const DeltaChainLink getlink(const size_t idx) { - return DeltaChainLink(&(_chain.delta_chain_links[idx])); - } - - size_t linkcount() { - return _chain.links_count; - } - - get_delta_chain_code_t code() { - return _chain.code; - } - -}; - -class DeltaChainIterator { - private: - size_t _index; - protected: - std::vector< std::shared_ptr > _chains; - DeltaChainIterator() : - _index(0) {} - virtual std::shared_ptr getNextChain(const Key &key) { - return std::make_shared(GET_DELTA_CHAIN_NOT_FOUND); - } - public: - DeltaChainIterator(std::shared_ptr chain) : - _index(0) { - _chains.push_back(chain); - } - - DeltaChainLink next() { - std::shared_ptr chain = _chains.back(); - - if (_index >= chain->linkcount()) { - // If we're not at the end, and we have a callback to fetch more, do the - // fetch. - bool refreshed = false; - if (chain->linkcount() > 0) { - DeltaChainLink result = chain->getlink(_index - 1); - - const uint8_t *deltabasenode = result.deltabasenode(); - if (memcmp(deltabasenode, NULLID, BIN_NODE_SIZE) != 0) { - Key key(result.filename(), result.filenamesz(), - (const char*)deltabasenode, BIN_NODE_SIZE); - - std::shared_ptr newChain = this->getNextChain(key); - if (newChain->code() == GET_DELTA_CHAIN_OK) { - // Do not free the old chain, since the iterator consumer may - // still be holding references to it. - _chains.push_back(newChain); - chain = _chains.back(); - _index = 0; - refreshed = true; - } - } - } - - if (!refreshed) { - return DeltaChainLink(NULL); - } - } - - DeltaChainLink result = chain->getlink(_index); - _index++; - - return result; - } -}; +#include "cstore/deltachain.h" +#include "cstore/key.h" class DataStore { protected: diff --git a/cstore/deltachain.h b/cstore/deltachain.h new file mode 100644 --- /dev/null +++ b/cstore/deltachain.h @@ -0,0 +1,215 @@ +// Copyright (c) 2004-present, Facebook, Inc. +// All Rights Reserved. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +// deltachain.h - c++ declaration of deltachain and related classes +// no-check-code + +#ifndef FBHGEXT_DELTACHAIN_H +#define FBHGEXT_DELTACHAIN_H + +extern "C" { +#include "cdatapack/cdatapack.h" +} + +#include +#include + +#include "cstore/key.h" + +/* + * Wrapper around delta chain link, both C and Python + */ +class DeltaChainLink { + private: + const char *_filename, *_deltabasefilename; + const uint8_t *_node, *_deltabasenode, *_delta; + uint16_t _filenamesz, _deltabasefilenamesz; + uint64_t _deltasz; + + public: + DeltaChainLink(delta_chain_link_t *link) { + if (link) { + _filename = link->filename; + _deltabasefilename = link->filename; + _node = link->node; + _deltabasenode = link->deltabase_node; + _delta = link->delta; + _filenamesz = link->filename_sz; + _deltabasefilenamesz = link->filename_sz; + _deltasz = link->delta_sz; + } else { + _filename = NULL; + _deltabasefilename = NULL; + _node = NULL; + _deltabasenode = NULL; + _delta = NULL; + _filenamesz = 0; + _deltabasefilenamesz = 0; + _deltasz = 0; + } + } + + DeltaChainLink(const char *filename, + const char *deltabasefilename, + const uint8_t *node, + const uint8_t *deltabasenode, + const uint8_t *delta, + uint16_t filenamesz, + uint16_t deltabasefilenamesz, + uint64_t deltasz) : + _filename(filename), + _deltabasefilename(deltabasefilename), + _node(node), + _deltabasenode(deltabasenode), + _delta(delta), + _filenamesz(filenamesz), + _deltabasefilenamesz(deltabasefilenamesz), + _deltasz(deltasz) {} + + ~DeltaChainLink() = default; + + const char* filename() { + return _filename; + } + + const char* deltabasefilename() { + return _deltabasefilename; + } + + const uint8_t* node() { + return _node; + } + + const uint8_t* deltabasenode() { + return _deltabasenode; + } + + const uint8_t* delta() { + return _delta; + } + + uint16_t filenamesz() { + return _filenamesz; + } + + uint16_t deltabasefilenamesz() { + return _deltabasefilenamesz; + } + + uint64_t deltasz() { + return _deltasz; + } + + bool isdone() { + return (_filename == NULL); + } +}; + +/* + * Abstract delta chain class + */ +class DeltaChain { + protected: + DeltaChain() {} + + public: + virtual ~DeltaChain() {} + + virtual const DeltaChainLink getlink(const size_t) = 0; + + virtual size_t linkcount() = 0; + + virtual get_delta_chain_code_t status() = 0; +}; + +/* + * Wrapper around C delta chain + * CDeltaChain takes ownership of delta_chain_t + */ +class CDeltaChain : public DeltaChain { + private: + delta_chain_t _chain; + + public: + // The constructor does a shallow copy of the delta chain and since the + // ownership is taken by this class it is responsible for memory management + CDeltaChain(delta_chain_t chain) : _chain(chain) {} + + CDeltaChain(get_delta_chain_code_t error) : + _chain(COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND }) {} + + // Memory of _chain has to be deallocated because it is a C struct that + // contains an array of delta_chain_link_t's + ~CDeltaChain() { + freedeltachain(_chain); + } + + const DeltaChainLink getlink(const size_t idx) { + return DeltaChainLink(&(_chain.delta_chain_links[idx])); + } + + size_t linkcount() { + return _chain.links_count; + } + + get_delta_chain_code_t status() { + return _chain.code; + } +}; + +/* + * Wrapper around python delta chain + */ +class PyDeltaChain : public DeltaChain { + private: + std::shared_ptr< std::vector > _chain; + + public: + PyDeltaChain(std::shared_ptr< std::vector > chain) : + _chain(chain) {} + + // Default destructor is used, because the destructor of _chain object + // will free the allocated memory automatically. + ~PyDeltaChain() {} + + const DeltaChainLink getlink(const size_t idx) { + return _chain->at(idx); + } + + size_t linkcount() { + return _chain->size(); + } + + get_delta_chain_code_t status() { + if (_chain->size()) { + return GET_DELTA_CHAIN_OK; + } else { + return GET_DELTA_CHAIN_NOT_FOUND; + } + } + +}; + +class DeltaChainIterator { + private: + size_t _index; + protected: + std::vector< std::shared_ptr > _chains; + DeltaChainIterator() : + _index(0) {} + virtual std::shared_ptr getNextChain(const Key &key) { + return std::make_shared(GET_DELTA_CHAIN_NOT_FOUND); + } + public: + DeltaChainIterator(std::shared_ptr chain) : + _index(0) { + _chains.push_back(chain); + } + + DeltaChainLink next(); +}; + +#endif // FBHGEXT_DELTACHAIN_H diff --git a/cstore/deltachain.cpp b/cstore/deltachain.cpp new file mode 100644 --- /dev/null +++ b/cstore/deltachain.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2004-present, Facebook, Inc. +// All Rights Reserved. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. + +// deltachain.h - c++ implementation of deltachain and related classes +// no-check-code + +#include "cstore/deltachain.h" + +DeltaChainLink DeltaChainIterator::next() { + std::shared_ptr chain = _chains.back(); + + if (_index >= chain->linkcount()) { + // If we're not at the end, and we have a callback to fetch more, do the + // fetch. + bool refreshed = false; + if (chain->linkcount() > 0) { + DeltaChainLink result = chain->getlink(_index - 1); + + const uint8_t *deltabasenode = result.deltabasenode(); + if (memcmp(deltabasenode, NULLID, BIN_NODE_SIZE) != 0) { + Key key(result.filename(), result.filenamesz(), + (const char*)deltabasenode, BIN_NODE_SIZE); + + std::shared_ptr newChain = this->getNextChain(key); + if (newChain->status() == GET_DELTA_CHAIN_OK) { + // Do not free the old chain, since the iterator consumer may + // still be holding references to it. + _chains.push_back(newChain); + chain = _chains.back(); + _index = 0; + refreshed = true; + } + } + } + + if (!refreshed) { + return DeltaChainLink(NULL); + } + } + + DeltaChainLink result = chain->getlink(_index); + _index++; + + return result; +} diff --git a/cstore/uniondatapackstore.cpp b/cstore/uniondatapackstore.cpp --- a/cstore/uniondatapackstore.cpp +++ b/cstore/uniondatapackstore.cpp @@ -93,7 +93,7 @@ DataStore *substore = *it; std::shared_ptr chain = substore->getDeltaChainRaw(key); - if (chain->code() == GET_DELTA_CHAIN_OK) { + if (chain->status() == GET_DELTA_CHAIN_OK) { return chain; } } diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -239,6 +239,7 @@ Extension('cstore', sources=[ 'cstore/datapackstore.cpp', + 'cstore/deltachain.cpp', 'cstore/py-cstore.cpp', 'cstore/pythonutil.cpp', 'cstore/uniondatapackstore.cpp', diff --git a/tests/test-check-code-hg.t b/tests/test-check-code-hg.t --- a/tests/test-check-code-hg.t +++ b/tests/test-check-code-hg.t @@ -65,6 +65,8 @@ Skipping cstore/datapackstore.cpp it has no-che?k-code (glob) Skipping cstore/datapackstore.h it has no-che?k-code (glob) Skipping cstore/datastore.h it has no-che?k-code (glob) + Skipping cstore/deltachain.cpp it has no-che?k-code (glob) + Skipping cstore/deltachain.h it has no-che?k-code (glob) Skipping cstore/key.h it has no-che?k-code (glob) Skipping cstore/match.h it has no-che?k-code (glob) Skipping cstore/py-cdatapack.h it has no-che?k-code (glob)