diff --git a/cstore/datastore.h b/cstore/datastore.h --- a/cstore/datastore.h +++ b/cstore/datastore.h @@ -15,6 +15,7 @@ } #include +#include #include #include "cstore/key.h" @@ -48,6 +49,17 @@ } } + DeltaChainLink(const char *filename, const char *deltabasefilename, + const uint8_t *node, const uint8_t *deltabasenode, + const uint8_t *delta) : + _filename(filename), + _deltabasefilename(deltabasefilename), + _node(node), + _deltabasenode(deltabasenode), + _delta(delta), + _filenamesz(sizeof *filename), + _deltasz(sizeof *delta) {} + const char* filename() { return _filename; } @@ -89,29 +101,56 @@ //C DeltaChain delta_chain_t _chain; + //Python DeltaChain + std::shared_ptr< std::vector > _pychain; + public: //The constructor does a shallow copy of the delta chain and since the //ownership is taken by this class it is responsible for memory management - DeltaChain(delta_chain_t chain) : _chain(chain) {} + DeltaChain(delta_chain_t chain) : + _chain(chain), + _pychain(NULL) {} - DeltaChain(get_delta_chain_code_t error) { - _chain = COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND }; - } + DeltaChain(get_delta_chain_code_t error) : + _chain(COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND }), + _pychain(NULL) {} + + DeltaChain(std::shared_ptr< std::vector > chain) : + _chain(COMPOUND_LITERAL(delta_chain_t) { GET_DELTA_CHAIN_NOT_FOUND }), + _pychain(chain) {} ~DeltaChain() { - freedeltachain(_chain); + if (_chain.code == GET_DELTA_CHAIN_OK) { + freedeltachain(_chain); + } } const DeltaChainLink getlink(const size_t idx) { - return DeltaChainLink(&(_chain.delta_chain_links[idx])); + if (_pychain) { + return _pychain->at(idx); + } else { + return DeltaChainLink(&(_chain.delta_chain_links[idx])); + } } size_t linkcount() { - return _chain.links_count; + if (_pychain) { + return _pychain->size(); + } else { + return _chain.links_count; + } } get_delta_chain_code_t code() { - return _chain.code; + if (_pychain) { + if (_pychain->size()) { + return GET_DELTA_CHAIN_OK; + } else { + return GET_DELTA_CHAIN_NOT_FOUND; + } + } else { + return _chain.code; + } } }; @@ -180,6 +219,8 @@ virtual DeltaChainIterator getDeltaChain(const Key &key) = 0; + virtual std::string type() {return "DataStore";} + virtual std::shared_ptr getDeltaChainRaw(const Key &key) = 0; virtual std::shared_ptr getMissing(KeyIterator &missing) = 0; diff --git a/cstore/py-datapackstore.h b/cstore/py-datapackstore.h --- a/cstore/py-datapackstore.h +++ b/cstore/py-datapackstore.h @@ -26,6 +26,8 @@ #include "cstore/key.h" #include "cstore/py-structs.h" #include "cstore/pythonutil.h" +#include "cstore/pythonkeyiterator.h" +#include "cstore/pythondatastore.h" #include "cstore/uniondatapackstore.h" // --------- DatapackStore Implementation --------- @@ -109,35 +111,6 @@ } } -class PythonKeyIterator : public KeyIterator { - private: - PythonObj _input; - Key _current; - public: - PythonKeyIterator(PythonObj input) : - _input(input) {} - - Key *next() { - PyObject *item; - while ((item = PyIter_Next((PyObject*)_input)) != NULL) { - PythonObj itemObj = item; - - char *name; - Py_ssize_t namelen; - char *node; - Py_ssize_t nodelen; - if (!PyArg_ParseTuple(item, "s#s#", &name, &namelen, &node, &nodelen)) { - throw pyexception(); - } - - _current = Key(name, namelen, node, nodelen); - return &_current; - } - - return NULL; - } -}; - static PyObject *datapackstore_getmissing(py_datapackstore *self, PyObject *keys) { try { PythonObj result = PyList_New(0); @@ -244,16 +217,14 @@ pySubStores.push_back(PythonObj(item)); int isinstance = PyObject_IsInstance(item, (PyObject*)&datapackstoreType); - if (isinstance == 0) { - PyErr_SetString(PyExc_RuntimeError, "cuniondatapackstore only accepts cdatapackstore"); - return -1; - } else if (isinstance != 1) { - // Error - return -1; + if (isinstance == 1) { + py_datapackstore *pySubStore = (py_datapackstore*)item; + stores.push_back(&pySubStore->datapackstore); + } else { + // Memory management of PythonDataStore object is done + // by UniondDatapackStore + stores.push_back(new PythonDataStore(item)); } - - py_datapackstore *pySubStore = (py_datapackstore*)item; - stores.push_back(&pySubStore->datapackstore); } // We have to manually call the member constructor, since the provided 'self' diff --git a/cstore/py-pythondatastore.h b/cstore/py-pythondatastore.h new file mode 100644 --- /dev/null +++ b/cstore/py-pythondatastore.h @@ -0,0 +1,79 @@ +// python implementation of a common interface +// +// Copyright 2017 Facebook, Inc. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. +// +// no-check-code + +// The PY_SSIZE_T_CLEAN define must be defined before the Python.h include, +// as per the documentation. + +#define PY_SSIZE_T_CLEAN +#include + +#include +#include + +#include "datastore.h" +#include "key.h" +#include "pythonutil.h" +#include "pythonkeyiterator.h" + +static std::shared_ptr pythonstore_getdeltachain(void *_store, + const Key &key) { + PythonObj store((PyObject*)_store); + // Build (name, node) tuple and call getdeltachain + // method of the underlying store + PythonObj pyKey = Py_BuildValue("(s#s#)", (key.name).c_str(), + (key.name).size(), key.node, 20); + PythonObj list = store.callmethod("getdeltachain", pyKey); + + // Extract the delta chain from the list of tuples + // and build a DeltaChain object from them + std::shared_ptr< std::vector > links = + std::make_shared< std::vector >(); + + PythonObj iter = PyObject_GetIter(list); + PyObject *tuple; + while ((tuple = PyIter_Next(iter)) != NULL) { + const char *filename, *deltabasefilename; + const uint8_t *node, *deltabasenode, *delta; + + int ok = PyArg_ParseTuple(tuple, "swsww", + &filename, &node, &deltabasefilename, + &deltabasenode, &delta); + + // the item is a five tuple + if (ok) { + links->push_back(DeltaChainLink(filename, deltabasefilename, + node, deltabasenode, delta)); + } + } + + return std::make_shared(links); +} + +static std::shared_ptr pythonstore_getmissing(void *_store, KeyIterator &missing) { + PythonObj store((PyObject*)_store); + PythonObj list = PyList_New(0); + + Key *key; + while ((key = missing.next()) != NULL) { + PythonObj pyKey = Py_BuildValue("(s#s#)", key->name.c_str(), + key->name.size(), key->node, 20); + if (PyList_Append(list, (PyObject*)pyKey)) { + return NULL; //throw an error + } + } + + PythonObj keys = store.callmethod("getmissing", list); + return std::make_shared(keys); +} + +static void pythonstore_markforrefresh(void *_store) { + PythonObj store((PyObject*)_store); + PythonObj args = Py_BuildValue(""); + store.callmethod("markforrefresh", args); +} diff --git a/cstore/pythondatastore.h b/cstore/pythondatastore.h new file mode 100644 --- /dev/null +++ b/cstore/pythondatastore.h @@ -0,0 +1,39 @@ +// pythondatastore.h - c++ declarations for a python data store +// +// Copyright 2017 Facebook, Inc. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. +// +// no-check-code + +#ifndef FBHGEXT_PYTHONDATASTORE_H +#define FBHGEXT_PYTHONDATASTORE_H + +#include + +#include "cstore/datastore.h" +#include "cstore/key.h" + +class PythonDataStore : public DataStore { + private: + void* _store; // pointer to python object + public: + PythonDataStore(void* store); + + ~PythonDataStore() = default; + + std::string type(); + + DeltaChainIterator getDeltaChain(const Key &key); + + std::shared_ptr getMissing(KeyIterator &missing); + + std::shared_ptr getDeltaChainRaw(const Key &key); + + bool contains(const Key &key); + + void markForRefresh(); +}; + +#endif //FBHGEXT_PYTHONDATASTORE_H diff --git a/cstore/pythondatastore.cpp b/cstore/pythondatastore.cpp new file mode 100644 --- /dev/null +++ b/cstore/pythondatastore.cpp @@ -0,0 +1,52 @@ +// pythondatastore.cpp - implementation of a python data store +// +// Copyright 2017 Facebook, Inc. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. +// +// no-check-code + +#include "py-pythondatastore.h" +#include "pythondatastore.h" + +PythonDataStore::PythonDataStore(void* store) : _store(store) {} + +std::string PythonDataStore::type() { + return "PythonDataStore"; +} + +DeltaChainIterator PythonDataStore::getDeltaChain(const Key &key) { + std::shared_ptr chain = pythonstore_getdeltachain(_store, key); + return DeltaChainIterator(chain); +} + +std::shared_ptr PythonDataStore::getDeltaChainRaw(const Key &key) { + return pythonstore_getdeltachain(_store, key); +} + +std::shared_ptr PythonDataStore::getMissing(KeyIterator &missing) { + return pythonstore_getmissing(_store, missing); +} + +void PythonDataStore::markForRefresh() { + pythonstore_markforrefresh(_store); +} + +class Single : public KeyIterator { + public: + Key *_k; + Single(Key *k) : _k(k) {} + Key *next() { + Key *tmp = _k; + _k = NULL; + return tmp; + } +}; + +bool PythonDataStore::contains(const Key &key) { + Key copy = key; + Single iter(©); + std::shared_ptr it = pythonstore_getmissing(_store, iter); + return (!it->next()); +} diff --git a/cstore/pythonkeyiterator.h b/cstore/pythonkeyiterator.h new file mode 100644 --- /dev/null +++ b/cstore/pythonkeyiterator.h @@ -0,0 +1,44 @@ +// pythonkeyiterator.h - c++ implementation of python key iterator +// +// Copyright 2017 Facebook, Inc. +// +// This software may be used and distributed according to the terms of the +// GNU General Public License version 2 or any later version. +// +// no-check-code + +#ifndef FBHGEXT_PYTHONKEYITERATOR_H +#define FBHGEXT_PYTHONKEYITERATOR_H + +#include "cstore/pythonutil.h" + +class PythonKeyIterator : public KeyIterator { + private: + PythonObj _input; + Key _current; + public: + PythonKeyIterator(PythonObj input) : + _input(input) {} + + Key *next() { + PyObject *item; + while ((item = PyIter_Next((PyObject*)_input)) != NULL) { + PythonObj itemObj = item; + + char *name; + Py_ssize_t namelen; + char *node; + Py_ssize_t nodelen; + if (!PyArg_ParseTuple(item, "s#s#", &name, &namelen, &node, &nodelen)) { + throw pyexception(); + } + + _current = Key(name, namelen, node, nodelen); + return &_current; + } + + return NULL; + } +}; + +#endif //FBHGEXT_PYTHONKEYITERATOR_H diff --git a/cstore/uniondatapackstore.cpp b/cstore/uniondatapackstore.cpp --- a/cstore/uniondatapackstore.cpp +++ b/cstore/uniondatapackstore.cpp @@ -24,6 +24,17 @@ // TODO: we should manage the substore lifetimes here, but because they are // also controlled by Python, we need to let python handle it and manage the // refcount in the py_uniondatapackstore type. + + // Only free memory of PythonDataStore objects + // Lifetime of others is managed by Python + for(std::vector::iterator it = _stores.begin(); + it != _stores.end(); + it++) { + DataStore *substore = *it; + if (substore->type() == "PythonDataStore") { + delete substore; + } + } } mpatch_flist* getNextLink(void* container, ssize_t index) { diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -241,6 +241,7 @@ 'cstore/datapackstore.cpp', 'cstore/py-cstore.cpp', 'cstore/pythonutil.cpp', + 'cstore/pythondatastore.cpp', 'cstore/uniondatapackstore.cpp', 'ctreemanifest/manifest.cpp', 'ctreemanifest/manifest_entry.cpp', diff --git a/tests/test-check-code-hg.t b/tests/test-check-code-hg.t --- a/tests/test-check-code-hg.t +++ b/tests/test-check-code-hg.t @@ -70,8 +70,12 @@ Skipping cstore/py-cdatapack.h it has no-che?k-code (glob) Skipping cstore/py-cstore.cpp it has no-che?k-code (glob) Skipping cstore/py-datapackstore.h it has no-che?k-code (glob) + Skipping cstore/py-pythondatastore.h it has no-che?k-code (glob) Skipping cstore/py-structs.h it has no-che?k-code (glob) Skipping cstore/py-treemanifest.h it has no-che?k-code (glob) + Skipping cstore/pythondatastore.cpp it has no-che?k-code (glob) + Skipping cstore/pythondatastore.h it has no-che?k-code (glob) + Skipping cstore/pythonkeyiterator.h it has no-che?k-code (glob) Skipping cstore/pythonutil.cpp it has no-che?k-code (glob) Skipping cstore/pythonutil.h it has no-che?k-code (glob) Skipping cstore/store.h it has no-che?k-code (glob)