( )⚙ D11610 largefiles: pass current transaction to `lfdirstate.write()`

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D11610

largefiles: pass current transaction to `lfdirstate.write()`
ClosedPublic

Authored by pulkit on Oct 5 2021, 7:43 AM.

Details

Reviewers
martinvonz
Group Reviewers
hg-reviewers
Commits
rHGdf3021c1f093: largefiles: pass current transaction to `lfdirstate.write()`
Summary

Right now, the largefile dirstate is not included in transaction which makes
things complex. Next patch will add code to do so, so let's make it mandatory to
pass current transaction and pass from all existing callers.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

pulkit created this revision.Oct 5 2021, 7:43 AM
martinvonz accepted this revision.Oct 8 2021, 4:53 PM
This revision is now accepted and ready to land.Oct 8 2021, 4:53 PM

Revision Contents
Changeset List

Diff 30670

hgext/largefiles/lfcommands.py

if ( if (
wvfs.exists(lfile) wvfs.exists(lfile)
and repo.dirstate.normalize(lfile) not in wctx and repo.dirstate.normalize(lfile) not in wctx
): ):
wvfs.unlinkpath(lfile) wvfs.unlinkpath(lfile)
removed += 1 removed += 1
# largefile processing might be slow and be interrupted - be prepared # largefile processing might be slow and be interrupted - be prepared
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
if lfiles: if lfiles:
lfiles = [f for f in lfiles if f not in dropped] lfiles = [f for f in lfiles if f not in dropped]
for f in dropped: for f in dropped:
repo.wvfs.unlinkpath(lfutil.standin(f)) repo.wvfs.unlinkpath(lfutil.standin(f))
# This needs to happen for dropped files, otherwise they stay in # This needs to happen for dropped files, otherwise they stay in
# the M state. # the M state.
mode |= (mode >> 2) & 0o111 & ~util.umask mode |= (mode >> 2) & 0o111 & ~util.umask
wvfs.chmod(lfile, mode) wvfs.chmod(lfile, mode)
update1 = 1 update1 = 1
updated += update1 updated += update1
lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
if lfiles: if lfiles:
statuswriter( statuswriter(
_(b'%d largefiles updated, %d removed\n') % (updated, removed) _(b'%d largefiles updated, %d removed\n') % (updated, removed)
) )
@eh.command( @eh.command(
b'lfpull', b'lfpull',

hgext/largefiles/lfutil.py

return super(largefilesdirstate, self).forget(unixpath(f)) return super(largefilesdirstate, self).forget(unixpath(f))
def normallookup(self, f): def normallookup(self, f):
return super(largefilesdirstate, self).normallookup(unixpath(f)) return super(largefilesdirstate, self).normallookup(unixpath(f))
def _ignore(self, f): def _ignore(self, f):
return False return False
def write(self, tr=False): def write(self, tr):
# (1) disable PENDING mode always # (1) disable PENDING mode always
# (lfdirstate isn't yet managed as a part of the transaction) # (lfdirstate isn't yet managed as a part of the transaction)
# (2) avoid develwarn 'use dirstate.write with ....' # (2) avoid develwarn 'use dirstate.write with ....'
super(largefilesdirstate, self).write(None) super(largefilesdirstate, self).write(None)
def openlfdirstate(ui, repo, create=True): def openlfdirstate(ui, repo, create=True):
""" """
# - are marked as "a" by "patch.patch()" (e.g. via transplant), and # - are marked as "a" by "patch.patch()" (e.g. via transplant), and
# - have to be marked as "n" after commit, but # - have to be marked as "n" after commit, but
# - aren't listed in "repo[node].files()" # - aren't listed in "repo[node].files()"
for f in ctx.files(): for f in ctx.files():
lfile = splitstandin(f) lfile = splitstandin(f)
if lfile is not None: if lfile is not None:
synclfdirstate(repo, lfdirstate, lfile, False) synclfdirstate(repo, lfdirstate, lfile, False)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
# As part of committing, copy all of the largefiles into the cache. # As part of committing, copy all of the largefiles into the cache.
# #
# Using "node" instead of "ctx" implies additional "repo[node]" # Using "node" instead of "ctx" implies additional "repo[node]"
# lookup while copyalltostore(), but can omit redundant check for # lookup while copyalltostore(), but can omit redundant check for
# files comming from the 2nd parent, which should exist in store # files comming from the 2nd parent, which should exist in store
# at merging. # at merging.
copyalltostore(repo, node) copyalltostore(repo, node)

hgext/largefiles/overrides.py

lfutil.writestandin( lfutil.writestandin(
repo, repo,
standinname, standinname,
hash=b'', hash=b'',
executable=lfutil.getexecutable(repo.wjoin(f)), executable=lfutil.getexecutable(repo.wjoin(f)),
) )
standins.append(standinname) standins.append(standinname)
lfdirstate.set_tracked(f) lfdirstate.set_tracked(f)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
bad += [ bad += [
lfutil.splitstandin(f) lfutil.splitstandin(f)
for f in repo[None].add(standins) for f in repo[None].add(standins)
if f in m.files() if f in m.files()
] ]
added = [f for f in lfnames if f not in bad] added = [f for f in lfnames if f not in bad]
return added, bad return added, bad
if not isaddremove: if not isaddremove:
for f in remove: for f in remove:
repo.wvfs.unlinkpath(f, ignoremissing=True) repo.wvfs.unlinkpath(f, ignoremissing=True)
repo[None].forget(remove) repo[None].forget(remove)
for f in remove: for f in remove:
lfdirstate.set_untracked(lfutil.splitstandin(f)) lfdirstate.set_untracked(lfutil.splitstandin(f))
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
return result return result
# For overriding mercurial.hgweb.webcommands so that largefiles will # For overriding mercurial.hgweb.webcommands so that largefiles will
# appear at their right place in the manifests. # appear at their right place in the manifests.
@eh.wrapfunction(webcommands, b'decodepath') @eh.wrapfunction(webcommands, b'decodepath')
def decodepath(orig, path): def decodepath(orig, path):
]: ]:
# this should be executed before 'orig', to execute 'remove' # this should be executed before 'orig', to execute 'remove'
# before all other actions # before all other actions
repo.dirstate.update_file( repo.dirstate.update_file(
lfile, p1_tracked=True, wc_tracked=False lfile, p1_tracked=True, wc_tracked=False
) )
# make sure lfile doesn't get synclfdirstate'd as normal # make sure lfile doesn't get synclfdirstate'd as normal
lfdirstate.update_file(lfile, p1_tracked=False, wc_tracked=True) lfdirstate.update_file(lfile, p1_tracked=False, wc_tracked=True)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
return orig(repo, actions, branchmerge, getfiledata) return orig(repo, actions, branchmerge, getfiledata)
# Override filemerge to prompt the user about how they wish to merge # Override filemerge to prompt the user about how they wish to merge
# largefiles. This will handle identical edits without prompting the user. # largefiles. This will handle identical edits without prompting the user.
@eh.wrapfunction(filemerge, b'_filemerge') @eh.wrapfunction(filemerge, b'_filemerge')
def overridefilemerge( def overridefilemerge(
# The file is gone, but this deletes any empty parent # The file is gone, but this deletes any empty parent
# directories as a side-effect. # directories as a side-effect.
repo.wvfs.unlinkpath(srclfile, ignoremissing=True) repo.wvfs.unlinkpath(srclfile, ignoremissing=True)
lfdirstate.set_untracked(srclfile) lfdirstate.set_untracked(srclfile)
else: else:
util.copyfile(repo.wjoin(srclfile), repo.wjoin(destlfile)) util.copyfile(repo.wjoin(srclfile), repo.wjoin(destlfile))
lfdirstate.set_tracked(destlfile) lfdirstate.set_tracked(destlfile)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
except error.Abort as e: except error.Abort as e:
if e.message != _(b'no files to copy'): if e.message != _(b'no files to copy'):
raise e raise e
else: else:
nolfiles = True nolfiles = True
finally: finally:
wlock.release() wlock.release()
@eh.wrapfunction(cmdutil, b'revert') @eh.wrapfunction(cmdutil, b'revert')
def overriderevert(orig, ui, repo, ctx, *pats, **opts): def overriderevert(orig, ui, repo, ctx, *pats, **opts):
# Because we put the standins in a bad state (by updating them) # Because we put the standins in a bad state (by updating them)
# and then return them to a correct state we need to lock to # and then return them to a correct state we need to lock to
# prevent others from changing them in their incorrect state. # prevent others from changing them in their incorrect state.
with repo.wlock(): with repo.wlock():
lfdirstate = lfutil.openlfdirstate(ui, repo) lfdirstate = lfutil.openlfdirstate(ui, repo)
s = lfutil.lfdirstatestatus(lfdirstate, repo) s = lfutil.lfdirstatestatus(lfdirstate, repo)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
for lfile in s.modified: for lfile in s.modified:
lfutil.updatestandin(repo, lfile, lfutil.standin(lfile)) lfutil.updatestandin(repo, lfile, lfutil.standin(lfile))
for lfile in s.deleted: for lfile in s.deleted:
fstandin = lfutil.standin(lfile) fstandin = lfutil.standin(lfile)
if repo.wvfs.exists(fstandin): if repo.wvfs.exists(fstandin):
repo.wvfs.unlink(fstandin) repo.wvfs.unlink(fstandin)
oldstandins = lfutil.getstandinsstate(repo) oldstandins = lfutil.getstandinsstate(repo)
ui.status(_(b'removing %s\n') % uipathfn(f)) ui.status(_(b'removing %s\n') % uipathfn(f))
# Need to lock because standin files are deleted then removed from the # Need to lock because standin files are deleted then removed from the
# repository and we could race in-between. # repository and we could race in-between.
with repo.wlock(): with repo.wlock():
lfdirstate = lfutil.openlfdirstate(ui, repo) lfdirstate = lfutil.openlfdirstate(ui, repo)
for f in forget: for f in forget:
lfdirstate.set_untracked(f) lfdirstate.set_untracked(f)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
standins = [lfutil.standin(f) for f in forget] standins = [lfutil.standin(f) for f in forget]
for f in standins: for f in standins:
repo.wvfs.unlinkpath(f, ignoremissing=True) repo.wvfs.unlinkpath(f, ignoremissing=True)
rejected = repo[None].forget(standins) rejected = repo[None].forget(standins)
bad.extend(f for f in rejected if f in m.files()) bad.extend(f for f in rejected if f in m.files())
forgot.extend(f for f in forget if f not in rejected) forgot.extend(f for f in forget if f not in rejected)
return bad, forgot return bad, forgot
# in this case, content of standin file is meaningless # in this case, content of standin file is meaningless
# (in dctx, lfile is unknown, or normal file) # (in dctx, lfile is unknown, or normal file)
continue continue
lfutil.updatestandin(repo, lfile, fstandin) lfutil.updatestandin(repo, lfile, fstandin)
# mark all clean largefiles as dirty, just in case the update gets # mark all clean largefiles as dirty, just in case the update gets
# interrupted before largefiles and lfdirstate are synchronized # interrupted before largefiles and lfdirstate are synchronized
for lfile in oldclean: for lfile in oldclean:
lfdirstate.set_possibly_dirty(lfile) lfdirstate.set_possibly_dirty(lfile)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
oldstandins = lfutil.getstandinsstate(repo) oldstandins = lfutil.getstandinsstate(repo)
wc = kwargs.get('wc') wc = kwargs.get('wc')
if wc and wc.isinmemory(): if wc and wc.isinmemory():
# largefiles is not a good candidate for in-memory merge (large # largefiles is not a good candidate for in-memory merge (large
# files, custom dirstate, matcher usage). # files, custom dirstate, matcher usage).
raise error.ProgrammingError( raise error.ProgrammingError(
b'largefiles is not compatible with in-memory merge' b'largefiles is not compatible with in-memory merge'
) )
with lfdirstate.parentchange(): with lfdirstate.parentchange():
result = orig(repo, node, branchmerge, force, *args, **kwargs) result = orig(repo, node, branchmerge, force, *args, **kwargs)
newstandins = lfutil.getstandinsstate(repo) newstandins = lfutil.getstandinsstate(repo)
filelist = lfutil.getlfilestoupdate(oldstandins, newstandins) filelist = lfutil.getlfilestoupdate(oldstandins, newstandins)
# to avoid leaving all largefiles as dirty and thus rehash them, mark # to avoid leaving all largefiles as dirty and thus rehash them, mark
# all the ones that didn't change as clean # all the ones that didn't change as clean
for lfile in oldclean.difference(filelist): for lfile in oldclean.difference(filelist):
lfdirstate.update_file(lfile, p1_tracked=True, wc_tracked=True) lfdirstate.update_file(lfile, p1_tracked=True, wc_tracked=True)
lfdirstate.write() lfdirstate.write(repo.currenttransaction())
if branchmerge or force or partial: if branchmerge or force or partial:
filelist.extend(s.deleted + s.removed) filelist.extend(s.deleted + s.removed)
lfcommands.updatelfiles( lfcommands.updatelfiles(
repo.ui, repo, filelist=filelist, normallookup=partial repo.ui, repo, filelist=filelist, normallookup=partial
) )

hgext/largefiles/reposetup.py

] ]
else: # not against working directory else: # not against working directory
result = [ result = [
[lfutil.splitstandin(f) or f for f in items] [lfutil.splitstandin(f) or f for f in items]
for items in result for items in result
] ]
if gotlock: if gotlock:
lfdirstate.write() lfdirstate.write(self.currenttransaction())
self.lfstatus = True self.lfstatus = True
return scmutil.status(*result) return scmutil.status(*result)
def commitctx(self, ctx, *args, **kwargs): def commitctx(self, ctx, *args, **kwargs):
node = super(lfilesrepo, self).commitctx(ctx, *args, **kwargs) node = super(lfilesrepo, self).commitctx(ctx, *args, **kwargs)
class lfilesctx(ctx.__class__): class lfilesctx(ctx.__class__):