This is an archive of the discontinued Mercurial Phabricator instance.

Differential D282

sha1: provide wrapper functions around the SHA-1 implementation
ClosedPublic

Authored by simpkins on Aug 8 2017, 4:20 PM.
Tags
None
Subscribers
None

Details

Reviewers
ryanmce
Group Reviewers
Restricted Project
Commits
rFBHGXaac32a02272f: sha1: provide wrapper functions around the SHA-1 implementation
Summary

Add a clib/sha1.h file with SHA-1 utility functions that hide the details of
the underlying SHA-1 implementation being used. This will make it easier in
the future if we want to use the faster SHA-1 implementation from OpenSSL if it
is available, but fall back to the sha1collisiondetection library if it is not
available.

Test Plan

Confirmed the code builds and passes unit tests.

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

simpkins created this revision.Aug 8 2017, 4:20 PM
Herald added a reviewer: Restricted Project. · View Herald TranscriptAug 8 2017, 4:20 PM
ryanmce accepted this revision.Aug 9 2017, 5:57 AM
This revision is now accepted and ready to land.Aug 9 2017, 5:57 AM
Closed by commit rFBHGXaac32a02272f: sha1: provide wrapper functions around the SHA-1 implementation (authored by simpkins). · Explain WhyAug 22 2017, 10:21 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mcdatapack/cdatapack_get.c (12 lines)
Mcfastmanifest/checksum.c (15 lines)
AMclib/sha1.h (38 lines)
Mctreemanifest/manifest.cpp (19 lines)
Msetup.py (2 lines)
Mtests/test-check-code-hg.t (1 line)

Diff 1172

cdatapack/cdatapack_get.c

// Copyright 2016-present Facebook. All Rights Reserved. // Copyright 2016-present Facebook. All Rights Reserved.
// //
// cdatapack_get.c: Use the index to dump a node's delta chain. // cdatapack_get.c: Use the index to dump a node's delta chain.
// //
// no-check-code // no-check-code
#include <inttypes.h> #include <inttypes.h>
#include <memory.h> #include <memory.h>
#include <sha1dc/sha1.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include "convert.h"
#include "cdatapack.h" #include "cdatapack.h"
#include "clib/sha1.h"
#include "convert.h"
#define DATAIDX_EXT ".dataidx" #define DATAIDX_EXT ".dataidx"
#define DATAPACK_EXT ".datapack" #define DATAPACK_EXT ".datapack"
// if the platform uses XMM registers to run strlen, then we might load // if the platform uses XMM registers to run strlen, then we might load
// beyond the memory region allocated for the strings. to ensure we keep // beyond the memory region allocated for the strings. to ensure we keep
// valgrind happy, we pad the strings with extra space and initialize the areas. // valgrind happy, we pad the strings with extra space and initialize the areas.
#define XMM_SZ 16 #define XMM_SZ 16
// does not find the memory beyond our string uninitialized. // does not find the memory beyond our string uninitialized.
memset(&node_buffer[NODE_SZ * 2], 0, XMM_SZ); memset(&node_buffer[NODE_SZ * 2], 0, XMM_SZ);
memset(&deltabase_buffer[NODE_SZ * 2], 0, XMM_SZ); memset(&deltabase_buffer[NODE_SZ * 2], 0, XMM_SZ);
memset(&sha_buffer[NODE_SZ * 2], 0, XMM_SZ); memset(&sha_buffer[NODE_SZ * 2], 0, XMM_SZ);
for (int ix = 0; ix < chain.links_count; ix ++) { for (int ix = 0; ix < chain.links_count; ix ++) {
delta_chain_link_t *link = &chain.delta_chain_links[ix]; delta_chain_link_t *link = &chain.delta_chain_links[ix];
SHA1_CTX ctx; fbhg_sha1_ctx_t ctx;
SHA1DCInit(&ctx); fbhg_sha1_init(&ctx);
SHA1DCUpdate(&ctx, link->delta, link->delta_sz); fbhg_sha1_update(&ctx, link->delta, link->delta_sz);
SHA1DCFinal(sha, &ctx); fbhg_sha1_final(sha, &ctx);
if (last_filename_sz != link->filename_sz || if (last_filename_sz != link->filename_sz ||
memcmp(last_filename, link->filename, last_filename_sz) != 0) { memcmp(last_filename, link->filename, last_filename_sz) != 0) {
// print the filename // print the filename
printf("\n%-.*s\n", link->filename_sz, link->filename); printf("\n%-.*s\n", link->filename_sz, link->filename);
last_filename_sz = link->filename_sz; last_filename_sz = link->filename_sz;
last_filename = link->filename; last_filename = link->filename;
} }

cfastmanifest/checksum.c

// Copyright 2016-present Facebook. All Rights Reserved. // Copyright 2016-present Facebook. All Rights Reserved.
// //
// checksum.c: implementation for recalculating the checksums for // checksum.c: implementation for recalculating the checksums for
// intermediate nodes in a tree. // intermediate nodes in a tree.
// //
// no-check-code // no-check-code
#include <sha1dc/sha1.h> #include "clib/sha1.h"
#include "node.h" #include "node.h"
#include "tree.h" #include "tree.h"
static void update_checksum(node_t *node) { static void update_checksum(node_t *node) {
SHA1_CTX ctx; fbhg_sha1_ctx_t ctx;
SHA1DCInit(&ctx); fbhg_sha1_init(&ctx);
// find all the children and make sure their checksums are up-to-date. // find all the children and make sure their checksums are up-to-date.
for (child_num_t ix = 0; ix < node->num_children; ix++) { for (child_num_t ix = 0; ix < node->num_children; ix++) {
node_t* child = get_child_by_index(node, ix); node_t* child = get_child_by_index(node, ix);
if (child->checksum_valid == false) { if (child->checksum_valid == false) {
update_checksum(child); update_checksum(child);
} }
SHA1DCUpdate(&ctx, (const unsigned char*) child->name, child->name_sz); fbhg_sha1_update(&ctx, child->name, child->name_sz);
SHA1DCUpdate(&ctx, child->checksum, child->checksum_sz); fbhg_sha1_update(&ctx, child->checksum, child->checksum_sz);
SHA1DCUpdate(&ctx, &child->flags, 1); fbhg_sha1_update(&ctx, &child->flags, 1);
} }
SHA1DCFinal(node->checksum, &ctx); fbhg_sha1_final(node->checksum, &ctx);
node->checksum_sz = SHA1_BYTES; node->checksum_sz = SHA1_BYTES;
node->checksum_valid = true; node->checksum_valid = true;
} }
void update_checksums(tree_t *tree) { void update_checksums(tree_t *tree) {
update_checksum(tree->shadow_root); update_checksum(tree->shadow_root);
} }

clib/sha1.h

  • This file was added.
// Copyright (c) 2017-present, Facebook, Inc.
// All Rights Reserved.
//
// This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version.
// sha1.h - wrapper functions around the underlying SHA-1 implementation.
//
// no-check-code
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
#include <stdlib.h>
#include <sha1dc/sha1.h>
typedef SHA1_CTX fbhg_sha1_ctx_t;
static inline int fbhg_sha1_init(fbhg_sha1_ctx_t* ctx) {
SHA1DCInit(ctx);
return 0;
}
static inline int
fbhg_sha1_update(fbhg_sha1_ctx_t* ctx, const void* data, unsigned long length) {
SHA1DCUpdate(ctx, (const unsigned char*)data, length);
return 0;
}
static inline int fbhg_sha1_final(unsigned char* md, fbhg_sha1_ctx_t* ctx) {
return SHA1DCFinal(md, ctx);
}
#ifdef __cplusplus
} /* extern C */
#endif

ctreemanifest/manifest.cpp

// manifest.cpp - c++ implementation of a single manifest // manifest.cpp - c++ implementation of a single manifest
// //
// Copyright 2016 Facebook, Inc. // Copyright 2016 Facebook, Inc.
// //
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
// //
// no-check-code // no-check-code
#include <sha1dc/sha1.h>
#include "manifest.h" #include "manifest.h"
#include "clib/sha1.h"
Manifest::Manifest(ConstantStringRef &rawobj, const char *node) : Manifest::Manifest(ConstantStringRef &rawobj, const char *node) :
_rawobj(rawobj), _rawobj(rawobj),
_refcount(0), _refcount(0),
_mutable(false) { _mutable(false) {
const char *parseptr = _rawobj.content(); const char *parseptr = _rawobj.content();
const char *endptr = parseptr + _rawobj.size(); const char *endptr = parseptr + _rawobj.size();
result.append("\n", 1); result.append("\n", 1);
} }
} }
void Manifest::computeNode(const char *p1, const char *p2, char *result) { void Manifest::computeNode(const char *p1, const char *p2, char *result) {
std::string content; std::string content;
this->serialize(content); this->serialize(content);
SHA1_CTX ctx; fbhg_sha1_ctx_t ctx;
SHA1DCInit(&ctx); fbhg_sha1_init(&ctx);
if (memcmp(p1, p2, BIN_NODE_SIZE) == -1) { if (memcmp(p1, p2, BIN_NODE_SIZE) == -1) {
SHA1DCUpdate(&ctx, (const unsigned char *) p1, BIN_NODE_SIZE); fbhg_sha1_update(&ctx, p1, BIN_NODE_SIZE);
SHA1DCUpdate(&ctx, (const unsigned char *) p2, BIN_NODE_SIZE); fbhg_sha1_update(&ctx, p2, BIN_NODE_SIZE);
} else { } else {
SHA1DCUpdate(&ctx, (const unsigned char *)p2, BIN_NODE_SIZE); fbhg_sha1_update(&ctx, p2, BIN_NODE_SIZE);
SHA1DCUpdate(&ctx, (const unsigned char *)p1, BIN_NODE_SIZE); fbhg_sha1_update(&ctx, p1, BIN_NODE_SIZE);
} }
SHA1DCUpdate(&ctx, (const unsigned char *)content.c_str(), content.size()); fbhg_sha1_update(&ctx, content.c_str(), content.size());
SHA1DCFinal((unsigned char*)result, &ctx); fbhg_sha1_final((unsigned char*)result, &ctx);
} }
ManifestPtr::ManifestPtr() : ManifestPtr::ManifestPtr() :
manifest(NULL) { manifest(NULL) {
} }
ManifestPtr::ManifestPtr(Manifest *manifest) : ManifestPtr::ManifestPtr(Manifest *manifest) :
manifest(manifest) { manifest(manifest) {

setup.py

'cstore/pythonutil.cpp', 'cstore/pythonutil.cpp',
'cstore/uniondatapackstore.cpp', 'cstore/uniondatapackstore.cpp',
'ctreemanifest/manifest.cpp', 'ctreemanifest/manifest.cpp',
'ctreemanifest/manifest_entry.cpp', 'ctreemanifest/manifest_entry.cpp',
'ctreemanifest/manifest_fetcher.cpp', 'ctreemanifest/manifest_fetcher.cpp',
'ctreemanifest/treemanifest.cpp', 'ctreemanifest/treemanifest.cpp',
], ],
include_dirs=[ include_dirs=[
'.',
'ctreemanifest', 'ctreemanifest',
'cdatapack', 'cdatapack',
'clib', 'clib',
'cstore', 'cstore',
'third-party', 'third-party',
] + include_dirs, ] + include_dirs,
library_dirs=[ library_dirs=[
'build/' + distutils_dir_name('lib'), 'build/' + distutils_dir_name('lib'),
'cfastmanifest/tree_convert.c', 'cfastmanifest/tree_convert.c',
'cfastmanifest/tree_copy.c', 'cfastmanifest/tree_copy.c',
'cfastmanifest/tree_diff.c', 'cfastmanifest/tree_diff.c',
'cfastmanifest/tree_disk.c', 'cfastmanifest/tree_disk.c',
'cfastmanifest/tree_iterator.c', 'cfastmanifest/tree_iterator.c',
'cfastmanifest/tree_path.c', 'cfastmanifest/tree_path.c',
], ],
include_dirs=[ include_dirs=[
'.',
'cfastmanifest', 'cfastmanifest',
'clib', 'clib',
'third-party', 'third-party',
] + include_dirs, ] + include_dirs,
library_dirs=library_dirs, library_dirs=library_dirs,
libraries=['sha1detectcoll'], libraries=['sha1detectcoll'],
extra_compile_args=filter(None, [ extra_compile_args=filter(None, [
STDC99, STDC99,

tests/test-check-code-hg.t

Skipping cfastmanifest/tree_iterator.h it has no-che?k-code (glob) Skipping cfastmanifest/tree_iterator.h it has no-che?k-code (glob)
Skipping cfastmanifest/tree_iterator_test.c it has no-che?k-code (glob) Skipping cfastmanifest/tree_iterator_test.c it has no-che?k-code (glob)
Skipping cfastmanifest/tree_path.c it has no-che?k-code (glob) Skipping cfastmanifest/tree_path.c it has no-che?k-code (glob)
Skipping cfastmanifest/tree_path.h it has no-che?k-code (glob) Skipping cfastmanifest/tree_path.h it has no-che?k-code (glob)
Skipping cfastmanifest/tree_test.c it has no-che?k-code (glob) Skipping cfastmanifest/tree_test.c it has no-che?k-code (glob)
Skipping clib/buffer.c it has no-che?k-code (glob) Skipping clib/buffer.c it has no-che?k-code (glob)
Skipping clib/buffer.h it has no-che?k-code (glob) Skipping clib/buffer.h it has no-che?k-code (glob)
Skipping clib/null_test.c it has no-che?k-code (glob) Skipping clib/null_test.c it has no-che?k-code (glob)
Skipping clib/sha1.h it has no-che?k-code (glob)
Skipping cstore/datapackstore.cpp it has no-che?k-code (glob) Skipping cstore/datapackstore.cpp it has no-che?k-code (glob)
Skipping cstore/datapackstore.h it has no-che?k-code (glob) Skipping cstore/datapackstore.h it has no-che?k-code (glob)
Skipping cstore/key.h it has no-che?k-code (glob) Skipping cstore/key.h it has no-che?k-code (glob)
Skipping cstore/match.h it has no-che?k-code (glob) Skipping cstore/match.h it has no-che?k-code (glob)
Skipping cstore/py-cdatapack.h it has no-che?k-code (glob) Skipping cstore/py-cdatapack.h it has no-che?k-code (glob)
Skipping cstore/py-cstore.cpp it has no-che?k-code (glob) Skipping cstore/py-cstore.cpp it has no-che?k-code (glob)
Skipping cstore/py-datapackstore.h it has no-che?k-code (glob) Skipping cstore/py-datapackstore.h it has no-che?k-code (glob)
Skipping cstore/py-structs.h it has no-che?k-code (glob) Skipping cstore/py-structs.h it has no-che?k-code (glob)