This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6084

packaging: split downloading code into own module
ClosedPublic

Authored by indygreg on Mar 7 2019, 7:49 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGc2237fe1359e: packaging: split downloading code into own module
Summary

As we will introduce more code to support packaging, it will be
useful to have download code in its own module.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.Mar 7 2019, 7:49 PM
Closed by commit rHGc2237fe1359e: packaging: split downloading code into own module (authored by indygreg). · Explain WhyMar 9 2019, 8:50 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Pcontrib/packaging/hgpackaging/downloads.py (14 lines)
Copied from contrib/packaging/hgpackaging/util.py
Pcontrib/packaging/hgpackaging/util.py (106 lines)
Copied to contrib/packaging/hgpackaging/downloads.py
Mcontrib/packaging/inno/build.py (4 lines)
Mtests/test-check-code.t (1 line)

Diff 14439

contrib/packaging/hgpackaging/downloads.py

  • This file was copied from contrib/packaging/hgpackaging/util.py.
# util.py - Common packaging utility code. # downloads.py - Code for downloading dependencies.
# #
# Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com> # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
# no-check-code because Python 3 native. # no-check-code because Python 3 native.
import gzip import gzip
import hashlib import hashlib
import pathlib import pathlib
import tarfile
import urllib.request import urllib.request
import zipfile
def hash_path(p: pathlib.Path): def hash_path(p: pathlib.Path):
h = hashlib.sha256() h = hashlib.sha256()
with p.open('rb') as fh: with p.open('rb') as fh:
while True: while True:
chunk = fh.read(65536) chunk = fh.read(65536)
url = entry['url'] url = entry['url']
local_name = local_name or url[url.rindex('/') + 1:] local_name = local_name or url[url.rindex('/') + 1:]
local_path = dest_path / local_name local_path = dest_path / local_name
download_to_path(url, local_path, entry['size'], entry['sha256']) download_to_path(url, local_path, entry['size'], entry['sha256'])
return local_path return local_path
def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path):
with tarfile.open(source, 'r') as tf:
tf.extractall(dest)
def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path):
with zipfile.ZipFile(source, 'r') as zf:
zf.extractall(dest)

contrib/packaging/hgpackaging/util.py

  • This file was copied to contrib/packaging/hgpackaging/downloads.py.
# util.py - Common packaging utility code. # util.py - Common packaging utility code.
# #
# Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com> # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
# no-check-code because Python 3 native. # no-check-code because Python 3 native.
import gzip
import hashlib
import pathlib import pathlib
import tarfile import tarfile
import urllib.request
import zipfile import zipfile
def hash_path(p: pathlib.Path):
h = hashlib.sha256()
with p.open('rb') as fh:
while True:
chunk = fh.read(65536)
if not chunk:
break
h.update(chunk)
return h.hexdigest()
class IntegrityError(Exception):
"""Represents an integrity error when downloading a URL."""
def secure_download_stream(url, size, sha256):
"""Securely download a URL to a stream of chunks.
If the integrity of the download fails, an IntegrityError is
raised.
"""
h = hashlib.sha256()
length = 0
with urllib.request.urlopen(url) as fh:
if not url.endswith('.gz') and fh.info().get('Content-Encoding') == 'gzip':
fh = gzip.GzipFile(fileobj=fh)
while True:
chunk = fh.read(65536)
if not chunk:
break
h.update(chunk)
length += len(chunk)
yield chunk
digest = h.hexdigest()
if length != size:
raise IntegrityError('size mismatch on %s: wanted %d; got %d' % (
url, size, length))
if digest != sha256:
raise IntegrityError('sha256 mismatch on %s: wanted %s; got %s' % (
url, sha256, digest))
def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str):
"""Download a URL to a filesystem path, possibly with verification."""
# We download to a temporary file and rename at the end so there's
# no chance of the final file being partially written or containing
# bad data.
print('downloading %s to %s' % (url, path))
if path.exists():
good = True
if path.stat().st_size != size:
print('existing file size is wrong; removing')
good = False
if good:
if hash_path(path) != sha256:
print('existing file hash is wrong; removing')
good = False
if good:
print('%s exists and passes integrity checks' % path)
return
path.unlink()
tmp = path.with_name('%s.tmp' % path.name)
try:
with tmp.open('wb') as fh:
for chunk in secure_download_stream(url, size, sha256):
fh.write(chunk)
except IntegrityError:
tmp.unlink()
raise
tmp.rename(path)
print('successfully downloaded %s' % url)
def download_entry(entry: dict, dest_path: pathlib.Path, local_name=None) -> pathlib.Path:
url = entry['url']
local_name = local_name or url[url.rindex('/') + 1:]
local_path = dest_path / local_name
download_to_path(url, local_path, entry['size'], entry['sha256'])
return local_path
def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path): def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path):
with tarfile.open(source, 'r') as tf: with tarfile.open(source, 'r') as tf:
tf.extractall(dest) tf.extractall(dest)
def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path): def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path):
with zipfile.ZipFile(source, 'r') as zf: with zipfile.ZipFile(source, 'r') as zf:
zf.extractall(dest) zf.extractall(dest)

contrib/packaging/inno/build.py

"""Build the Inno installer. """Build the Inno installer.
Build files will be placed in ``build_dir``. Build files will be placed in ``build_dir``.
py2exe's setup.py doesn't use setuptools. It doesn't have modern logic py2exe's setup.py doesn't use setuptools. It doesn't have modern logic
for finding the Python 2.7 toolchain. So, we require the environment for finding the Python 2.7 toolchain. So, we require the environment
to already be configured with an active toolchain. to already be configured with an active toolchain.
""" """
from hgpackaging.util import ( from hgpackaging.downloads import (
download_entry, download_entry,
)
from hgpackaging.util import (
extract_tar_to_directory, extract_tar_to_directory,
extract_zip_to_directory, extract_zip_to_directory,
) )
if not iscc.exists(): if not iscc.exists():
raise Exception('%s does not exist' % iscc) raise Exception('%s does not exist' % iscc)
if 'VCINSTALLDIR' not in os.environ: if 'VCINSTALLDIR' not in os.environ:

tests/test-check-code.t

#require test-repo #require test-repo
$ . "$TESTDIR/helpers-testrepo.sh" $ . "$TESTDIR/helpers-testrepo.sh"
$ check_code="$TESTDIR"/../contrib/check-code.py $ check_code="$TESTDIR"/../contrib/check-code.py
$ cd "$TESTDIR"/.. $ cd "$TESTDIR"/..
New errors are not allowed. Warnings are strongly discouraged. New errors are not allowed. Warnings are strongly discouraged.
(The writing "no-che?k-code" is for not skipping this file when checking.) (The writing "no-che?k-code" is for not skipping this file when checking.)
$ testrepohg locate \ $ testrepohg locate \
> -X contrib/python-zstandard \ > -X contrib/python-zstandard \
> -X hgext/fsmonitor/pywatchman \ > -X hgext/fsmonitor/pywatchman \
> -X mercurial/thirdparty \ > -X mercurial/thirdparty \
> | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false > | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false
Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob)
Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob)
Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob) Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob)
Skipping i18n/polib.py it has no-che?k-code (glob) Skipping i18n/polib.py it has no-che?k-code (glob)
Skipping mercurial/statprof.py it has no-che?k-code (glob) Skipping mercurial/statprof.py it has no-che?k-code (glob)
Skipping tests/badserverext.py it has no-che?k-code (glob) Skipping tests/badserverext.py it has no-che?k-code (glob)
@commands in debugcommands.py should be in alphabetical order. @commands in debugcommands.py should be in alphabetical order.