diff --git a/contrib/packaging/hgpackaging/util.py b/contrib/packaging/hgpackaging/downloads.py copy from contrib/packaging/hgpackaging/util.py copy to contrib/packaging/hgpackaging/downloads.py --- a/contrib/packaging/hgpackaging/util.py +++ b/contrib/packaging/hgpackaging/downloads.py @@ -1,4 +1,4 @@ -# util.py - Common packaging utility code. +# downloads.py - Code for downloading dependencies. # # Copyright 2019 Gregory Szorc # @@ -10,9 +10,7 @@ import gzip import hashlib import pathlib -import tarfile import urllib.request -import zipfile def hash_path(p: pathlib.Path): @@ -116,13 +114,3 @@ download_to_path(url, local_path, entry['size'], entry['sha256']) return local_path - - -def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path): - with tarfile.open(source, 'r') as tf: - tf.extractall(dest) - - -def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path): - with zipfile.ZipFile(source, 'r') as zf: - zf.extractall(dest) diff --git a/contrib/packaging/hgpackaging/util.py b/contrib/packaging/hgpackaging/util.py --- a/contrib/packaging/hgpackaging/util.py +++ b/contrib/packaging/hgpackaging/util.py @@ -7,117 +7,11 @@ # no-check-code because Python 3 native. -import gzip -import hashlib import pathlib import tarfile -import urllib.request import zipfile -def hash_path(p: pathlib.Path): - h = hashlib.sha256() - - with p.open('rb') as fh: - while True: - chunk = fh.read(65536) - if not chunk: - break - - h.update(chunk) - - return h.hexdigest() - - -class IntegrityError(Exception): - """Represents an integrity error when downloading a URL.""" - - -def secure_download_stream(url, size, sha256): - """Securely download a URL to a stream of chunks. - - If the integrity of the download fails, an IntegrityError is - raised. - """ - h = hashlib.sha256() - length = 0 - - with urllib.request.urlopen(url) as fh: - if not url.endswith('.gz') and fh.info().get('Content-Encoding') == 'gzip': - fh = gzip.GzipFile(fileobj=fh) - - while True: - chunk = fh.read(65536) - if not chunk: - break - - h.update(chunk) - length += len(chunk) - - yield chunk - - digest = h.hexdigest() - - if length != size: - raise IntegrityError('size mismatch on %s: wanted %d; got %d' % ( - url, size, length)) - - if digest != sha256: - raise IntegrityError('sha256 mismatch on %s: wanted %s; got %s' % ( - url, sha256, digest)) - - -def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str): - """Download a URL to a filesystem path, possibly with verification.""" - - # We download to a temporary file and rename at the end so there's - # no chance of the final file being partially written or containing - # bad data. - print('downloading %s to %s' % (url, path)) - - if path.exists(): - good = True - - if path.stat().st_size != size: - print('existing file size is wrong; removing') - good = False - - if good: - if hash_path(path) != sha256: - print('existing file hash is wrong; removing') - good = False - - if good: - print('%s exists and passes integrity checks' % path) - return - - path.unlink() - - tmp = path.with_name('%s.tmp' % path.name) - - try: - with tmp.open('wb') as fh: - for chunk in secure_download_stream(url, size, sha256): - fh.write(chunk) - except IntegrityError: - tmp.unlink() - raise - - tmp.rename(path) - print('successfully downloaded %s' % url) - - -def download_entry(entry: dict, dest_path: pathlib.Path, local_name=None) -> pathlib.Path: - url = entry['url'] - - local_name = local_name or url[url.rindex('/') + 1:] - - local_path = dest_path / local_name - download_to_path(url, local_path, entry['size'], entry['sha256']) - - return local_path - - def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path): with tarfile.open(source, 'r') as tf: tf.extractall(dest) diff --git a/contrib/packaging/inno/build.py b/contrib/packaging/inno/build.py --- a/contrib/packaging/inno/build.py +++ b/contrib/packaging/inno/build.py @@ -87,8 +87,10 @@ for finding the Python 2.7 toolchain. So, we require the environment to already be configured with an active toolchain. """ + from hgpackaging.downloads import ( + download_entry, + ) from hgpackaging.util import ( - download_entry, extract_tar_to_directory, extract_zip_to_directory, ) diff --git a/tests/test-check-code.t b/tests/test-check-code.t --- a/tests/test-check-code.t +++ b/tests/test-check-code.t @@ -12,6 +12,7 @@ > -X hgext/fsmonitor/pywatchman \ > -X mercurial/thirdparty \ > | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false + Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob) Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob) Skipping i18n/polib.py it has no-che?k-code (glob)