diff --git a/mercurial/hg.py b/mercurial/hg.py --- a/mercurial/hg.py +++ b/mercurial/hg.py @@ -24,7 +24,6 @@ from . import ( bookmarks, bundlerepo, - cacheutil, cmdutil, destutil, discovery, @@ -866,18 +865,10 @@ requirements=dest_reqs, ) destrepo = localrepo.makelocalrepository(ui, destrootpath) + destlock = destrepo.lock() + from . import streamclone # avoid cycle - destpath = destrepo.vfs.base - destlock = copystore(ui, srcrepo, destpath) - # copy bookmarks over - srcbookmarks = srcrepo.vfs.join(b'bookmarks') - dstbookmarks = os.path.join(destpath, b'bookmarks') - if os.path.exists(srcbookmarks): - util.copyfile(srcbookmarks, dstbookmarks) - - dstcachedir = os.path.join(destpath, b'cache') - for cache in cacheutil.cachetocopy(srcrepo): - _copycache(srcrepo, dstcachedir, cache) + streamclone.local_copy(srcrepo, destrepo) # we need to re-init the repo after manually copying the data # into it diff --git a/mercurial/streamclone.py b/mercurial/streamclone.py --- a/mercurial/streamclone.py +++ b/mercurial/streamclone.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import contextlib +import errno import os import struct @@ -15,6 +16,7 @@ from .pycompat import open from .interfaces import repository from . import ( + bookmarks, cacheutil, error, narrowspec, @@ -25,6 +27,9 @@ store, util, ) +from .utils import ( + stringutil, +) def canperformstreamclone(pullop, bundle2=False): @@ -771,3 +776,104 @@ repo.ui, repo.requirements, repo.features ) scmutil.writereporequirements(repo) + + +def _copy_files(src_vfs_map, dst_vfs_map, entries, progress): + hardlink = [True] + + def copy_used(): + hardlink[0] = False + progress.topic = _(b'copying') + + for k, path, size in entries: + src_vfs = src_vfs_map[k] + dst_vfs = dst_vfs_map[k] + src_path = src_vfs.join(path) + dst_path = dst_vfs.join(path) + dirname = dst_vfs.dirname(path) + if not dst_vfs.exists(dirname): + dst_vfs.makedirs(dirname) + dst_vfs.register_file(path) + # XXX we could use the #nb_bytes argument. + util.copyfile( + src_path, + dst_path, + hardlink=hardlink[0], + no_hardlink_cb=copy_used, + check_fs_hardlink=False, + ) + progress.increment() + return hardlink[0] + + +def local_copy(src_repo, dest_repo): + """copy all content from one local repository to another + + This is useful for local clone""" + src_store_requirements = { + r + for r in src_repo.requirements + if r not in requirementsmod.WORKING_DIR_REQUIREMENTS + } + dest_store_requirements = { + r + for r in dest_repo.requirements + if r not in requirementsmod.WORKING_DIR_REQUIREMENTS + } + assert src_store_requirements == dest_store_requirements + + with dest_repo.lock(): + with src_repo.lock(): + entries, totalfilesize = _v2_walk( + src_repo, + includes=None, + excludes=None, + includeobsmarkers=True, + ) + src_vfs_map = _makemap(src_repo) + dest_vfs_map = _makemap(dest_repo) + progress = src_repo.ui.makeprogress( + topic=_(b'linking'), + total=len(entries), + unit=_(b'files'), + ) + # copy files + # + # We could copy the full file while the source repository is locked + # and the other one without the lock. However, in the linking case, + # this would also requires checks that nobody is appending any data + # to the files while we do the clone, so this is not done yet. We + # could do this blindly when copying files. + files = ((k, path, size) for k, path, ftype, size in entries) + hardlink = _copy_files(src_vfs_map, dest_vfs_map, files, progress) + + # copy bookmarks over + src_book_vfs = bookmarks.bookmarksvfs(src_repo) + srcbookmarks = src_book_vfs.join(b'bookmarks') + dst_book_vfs = bookmarks.bookmarksvfs(dest_repo) + dstbookmarks = dst_book_vfs.join(b'bookmarks') + if os.path.exists(srcbookmarks): + util.copyfile(srcbookmarks, dstbookmarks) + progress.complete() + if hardlink: + msg = b'linked %d files\n' + else: + msg = b'copied %d files\n' + src_repo.ui.debug(msg % len(entries)) + + with dest_repo.transaction(b"localclone") as tr: + dest_repo.store.write(tr) + + # clean up transaction file as they do not make sense + undo_files = [(dest_repo.svfs, b'undo.backupfiles')] + undo_files.extend(dest_repo.undofiles()) + for undovfs, undofile in undo_files: + try: + undovfs.unlink(undofile) + except OSError as e: + if e.errno != errno.ENOENT: + msg = _(b'error removing %s: %s\n') + path = undovfs.join(undofile) + e_msg = stringutil.forcebytestr(e) + msg %= (path, e_msg) + dest_repo.ui.warn(msg) diff --git a/tests/test-clone.t b/tests/test-clone.t --- a/tests/test-clone.t +++ b/tests/test-clone.t @@ -86,26 +86,22 @@ #if hardlink $ hg --debug clone -U . ../c --config progress.debug=true - linking: 1 files - linking: 2 files - linking: 3 files - linking: 4 files - linking: 5 files - linking: 6 files - linking: 7 files - linking: 8 files - linked 8 files (reporevlogstore !) - linking: 9 files (reposimplestore !) - linking: 10 files (reposimplestore !) - linking: 11 files (reposimplestore !) - linking: 12 files (reposimplestore !) - linking: 13 files (reposimplestore !) - linking: 14 files (reposimplestore !) - linking: 15 files (reposimplestore !) - linking: 16 files (reposimplestore !) - linking: 17 files (reposimplestore !) - linking: 18 files (reposimplestore !) - linked 18 files (reposimplestore !) + linking: 1/15 files (6.67%) + linking: 2/15 files (13.33%) + linking: 3/15 files (20.00%) + linking: 4/15 files (26.67%) + linking: 5/15 files (33.33%) + linking: 6/15 files (40.00%) + linking: 7/15 files (46.67%) + linking: 8/15 files (53.33%) + linking: 9/15 files (60.00%) + linking: 10/15 files (66.67%) + linking: 11/15 files (73.33%) + linking: 12/15 files (80.00%) + linking: 13/15 files (86.67%) + linking: 14/15 files (93.33%) + linking: 15/15 files (100.00%) + linked 15 files updating the branch cache #else $ hg --debug clone -U . ../c --config progress.debug=true @@ -117,18 +113,6 @@ copying: 6 files copying: 7 files copying: 8 files - copied 8 files (reporevlogstore !) - copying: 9 files (reposimplestore !) - copying: 10 files (reposimplestore !) - copying: 11 files (reposimplestore !) - copying: 12 files (reposimplestore !) - copying: 13 files (reposimplestore !) - copying: 14 files (reposimplestore !) - copying: 15 files (reposimplestore !) - copying: 16 files (reposimplestore !) - copying: 17 files (reposimplestore !) - copying: 18 files (reposimplestore !) - copied 18 files (reposimplestore !) #endif $ cd ../c diff --git a/tests/test-empty.t b/tests/test-empty.t --- a/tests/test-empty.t +++ b/tests/test-empty.t @@ -45,6 +45,7 @@ $ ls .hg 00changelog.i cache + dirstate hgrc requires store diff --git a/tests/test-fncache.t b/tests/test-fncache.t --- a/tests/test-fncache.t +++ b/tests/test-fncache.t @@ -528,7 +528,11 @@ Unbundling should follow the same rules; existing files should not cause a load: +(loading during the clone is expected) $ hg clone -q . tobundle + fncache load triggered! + fncache load triggered! + $ echo 'new line' > tobundle/bar $ hg -R tobundle ci -qm bar $ hg -R tobundle bundle -q barupdated.hg diff --git a/tests/test-hardlinks.t b/tests/test-hardlinks.t --- a/tests/test-hardlinks.t +++ b/tests/test-hardlinks.t @@ -61,13 +61,13 @@ Create hardlinked clone r2: $ hg clone -U --debug r1 r2 --config progress.debug=true - linking: 1 files - linking: 2 files - linking: 3 files - linking: 4 files - linking: 5 files - linking: 6 files - linking: 7 files + linking: 1/7 files (14.29%) + linking: 2/7 files (28.57%) + linking: 3/7 files (42.86%) + linking: 4/7 files (57.14%) + linking: 5/7 files (71.43%) + linking: 6/7 files (85.71%) + linking: 7/7 files (100.00%) linked 7 files updating the branch cache @@ -91,7 +91,7 @@ 2 r1/.hg/store/00manifest.i 2 r1/.hg/store/data/d1/f2.i 2 r1/.hg/store/data/f1.i - 2 r1/.hg/store/fncache (repofncache !) + 1 r1/.hg/store/fncache (repofncache !) 1 r1/.hg/store/phaseroots 1 r1/.hg/store/undo 1 r1/.hg/store/undo.backup.fncache (repofncache !) @@ -103,7 +103,7 @@ 2 r2/.hg/store/00manifest.i 2 r2/.hg/store/data/d1/f2.i 2 r2/.hg/store/data/f1.i - 2 r2/.hg/store/fncache (repofncache !) + 1 r2/.hg/store/fncache (repofncache !) Repo r3 should not be hardlinked: @@ -175,7 +175,7 @@ #if hardlink-whitelisted repofncache $ nlinksdir r2/.hg/store/fncache - 2 r2/.hg/store/fncache + 1 r2/.hg/store/fncache #endif $ hg -R r2 verify @@ -201,11 +201,11 @@ 1 r2/.hg/store/00manifest.i 1 r2/.hg/store/data/d1/f2.i 1 r2/.hg/store/data/f1.i - [12] r2/\.hg/store/fncache (re) (repofncache !) + 1 r2/.hg/store/fncache (repofncache !) #if hardlink-whitelisted repofncache $ nlinksdir r2/.hg/store/fncache - 2 r2/.hg/store/fncache + 1 r2/.hg/store/fncache #endif Create a file which exec permissions we will change diff --git a/tests/test-subrepo-deep-nested-change.t b/tests/test-subrepo-deep-nested-change.t --- a/tests/test-subrepo-deep-nested-change.t +++ b/tests/test-subrepo-deep-nested-change.t @@ -28,12 +28,12 @@ $ echo "sub2 = ../sub2" > sub1/.hgsub $ hg clone sub2 sub1/sub2 \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) updating [===========================================>] 1/1\r (no-eol) (esc) @@ -52,27 +52,25 @@ $ echo "sub1 = ../sub1" > main/.hgsub $ hg clone sub1 main/sub1 \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) updating [===========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) updating [===========================================>] 1/1\r (no-eol) (esc) \r (no-eol) (esc) updating to branch default @@ -156,46 +154,36 @@ $ hg --config extensions.largefiles= clone main cloned \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) updating [===========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) updating [===========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 2\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 3\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 4\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 5\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 6\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 1\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 2\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 3\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 4\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 5\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 6\r (no-eol) (esc) (reposimplestore !) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) updating [===========================================>] 1/1\r (no-eol) (esc) \r (no-eol) (esc) updating to branch default diff --git a/tests/test-subrepo-recursion.t b/tests/test-subrepo-recursion.t --- a/tests/test-subrepo-recursion.t +++ b/tests/test-subrepo-recursion.t @@ -454,19 +454,15 @@ #if hardlink $ hg clone -U . ../empty \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 11\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 12\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 13\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/9\r (no-eol) (esc) + linking [=========> ] 2/9\r (no-eol) (esc) + linking [==============> ] 3/9\r (no-eol) (esc) + linking [===================> ] 4/9\r (no-eol) (esc) + linking [========================> ] 5/9\r (no-eol) (esc) + linking [=============================> ] 6/9\r (no-eol) (esc) + linking [==================================> ] 7/9\r (no-eol) (esc) + linking [=======================================> ] 8/9\r (no-eol) (esc) + linking [============================================>] 9/9\r (no-eol) (esc) \r (no-eol) (esc) #else $ hg clone -U . ../empty @@ -484,22 +480,14 @@ archiving [==========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 11\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 12\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 13\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 14\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 15\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 16\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) archiving (foo) [ ] 0/3\r (no-eol) (esc) @@ -508,15 +496,12 @@ archiving (foo) [====================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 8\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) archiving (foo/bar) [ ] 0/1\r (no-eol) (esc)