This is an archive of the discontinued Mercurial Phabricator instance.

Differential D2097

infinitepush: remove backupcommands.py
ClosedPublic

Authored by pulkit on Feb 9 2018, 6:45 AM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHGde4c2f3af97f: infinitepush: remove backupcommands.py
Summary

This file contains command pushbackup and pullbackup which are used to store
things to infinitepush.

This is an advanced functionality which we don't require as of now. Also this
uses shareutil.py from fb-hgext/ which is not there in core. Therefore this
patch deletes the file and the config option which belongs to the backup thing.

If we need this functionality in future, we can always move this file back.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

pulkit created this revision.Feb 9 2018, 6:45 AM
indygreg accepted this revision.Feb 9 2018, 10:03 PM
indygreg added a subscriber: indygreg.

This is a nifty feature. But I agree it is beyond scope of the initial landing of this feature.

This revision is now accepted and ready to land.Feb 9 2018, 10:03 PM
Closed by commit rHGde4c2f3af97f: infinitepush: remove backupcommands.py (authored by pulkit). · Explain WhyMar 30 2018, 3:51 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 7412

hgext/infinitepush/__init__.py

phases, phases,
pushkey, pushkey,
registrar, registrar,
util, util,
wireproto, wireproto,
) )
from . import ( from . import (
backupcommands,
bundleparts, bundleparts,
common, common,
infinitepushcommands, infinitepushcommands,
) )
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
# be specifying the version(s) of Mercurial they are tested with, or # be specifying the version(s) of Mercurial they are tested with, or
default='', default='',
) )
configitem('infinitepush', 'branchpattern', configitem('infinitepush', 'branchpattern',
default='', default='',
) )
configitem('infinitepush', 'metadatafilelimit', configitem('infinitepush', 'metadatafilelimit',
default=100, default=100,
) )
configitem('infinitepushbackup', 'autobackup',
default=False,
)
configitem('experimental', 'server-bundlestore-bookmark', configitem('experimental', 'server-bundlestore-bookmark',
default='', default='',
) )
configitem('experimental', 'server-bundlestore-create', configitem('experimental', 'server-bundlestore-create',
default='', default='',
) )
configitem('experimental', 'infinitepush-scratchpush', configitem('experimental', 'infinitepush-scratchpush',
default=False, default=False,
) )
configitem('experimental', 'non-forward-move', configitem('experimental', 'non-forward-move',
default=False, default=False,
) )
pushrebaseparttype = 'b2x:rebase' pushrebaseparttype = 'b2x:rebase'
experimental = 'experimental' experimental = 'experimental'
configbookmark = 'server-bundlestore-bookmark' configbookmark = 'server-bundlestore-bookmark'
configcreate = 'server-bundlestore-create' configcreate = 'server-bundlestore-create'
configscratchpush = 'infinitepush-scratchpush' configscratchpush = 'infinitepush-scratchpush'
confignonforwardmove = 'non-forward-move' confignonforwardmove = 'non-forward-move'
scratchbranchparttype = bundleparts.scratchbranchparttype scratchbranchparttype = bundleparts.scratchbranchparttype
cmdtable = infinitepushcommands.cmdtable cmdtable = infinitepushcommands.cmdtable
revsetpredicate = backupcommands.revsetpredicate revsetpredicate = registrar.revsetpredicate()
templatekeyword = backupcommands.templatekeyword templatekeyword = registrar.templatekeyword()
_scratchbranchmatcher = lambda x: False _scratchbranchmatcher = lambda x: False
_maybehash = re.compile(r'^[a-f0-9]+$').search _maybehash = re.compile(r'^[a-f0-9]+$').search
def _buildexternalbundlestore(ui): def _buildexternalbundlestore(ui):
put_args = ui.configlist('infinitepush', 'put_args', []) put_args = ui.configlist('infinitepush', 'put_args', [])
put_binary = ui.config('infinitepush', 'put_binary') put_binary = ui.config('infinitepush', 'put_binary')
if not put_binary: if not put_binary:
raise error.Abort('put binary is not specified') raise error.Abort('put binary is not specified')
# remotenames circumvents the default push implementation entirely, so make # remotenames circumvents the default push implementation entirely, so make
# sure we load after it so that we wrap it. # sure we load after it so that we wrap it.
order = extensions._order order = extensions._order
order.remove('infinitepush') order.remove('infinitepush')
order.append('infinitepush') order.append('infinitepush')
extensions._order = order extensions._order = order
def extsetup(ui): def extsetup(ui):
# Allow writing backup files outside the normal lock
localrepo.localrepository._wlockfreeprefix.update([
backupcommands._backupstatefile,
backupcommands._backupgenerationfile,
backupcommands._backuplatestinfofile,
])
commonsetup(ui) commonsetup(ui)
if _isserver(ui): if _isserver(ui):
serverextsetup(ui) serverextsetup(ui)
else: else:
clientextsetup(ui) clientextsetup(ui)
def commonsetup(ui): def commonsetup(ui):
wireproto.commands['listkeyspatterns'] = ( wireproto.commands['listkeyspatterns'] = (
# to avoid generation of both parts. # to avoid generation of both parts.
partorder = exchange.b2partsgenorder partorder = exchange.b2partsgenorder
index = partorder.index('changeset') index = partorder.index('changeset')
if pushrebaseparttype in partorder: if pushrebaseparttype in partorder:
index = min(index, partorder.index(pushrebaseparttype)) index = min(index, partorder.index(pushrebaseparttype))
partorder.insert( partorder.insert(
index, partorder.pop(partorder.index(scratchbranchparttype))) index, partorder.pop(partorder.index(scratchbranchparttype)))
def wrapsmartlog(loaded):
if not loaded:
return
smartlogmod = extensions.find('smartlog')
extensions.wrapcommand(smartlogmod.cmdtable, 'smartlog', _smartlog)
extensions.afterloaded('smartlog', wrapsmartlog)
backupcommands.extsetup(ui)
def _smartlog(orig, ui, repo, **opts):
res = orig(ui, repo, **opts)
backupcommands.smartlogsummary(ui, repo)
return res
def _showbookmarks(ui, bookmarks, **opts): def _showbookmarks(ui, bookmarks, **opts):
# Copy-paste from commands.py # Copy-paste from commands.py
fm = ui.formatter('bookmarks', opts) fm = ui.formatter('bookmarks', opts)
for bmark, n in sorted(bookmarks.iteritems()): for bmark, n in sorted(bookmarks.iteritems()):
fm.startitem() fm.startitem()
if not ui.quiet: if not ui.quiet:
fm.plain(' ') fm.plain(' ')
fm.write('bookmark', '%s', bmark) fm.write('bookmark', '%s', bmark)

hgext/infinitepush/backupcommands.py

  • This file was deleted.
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
"""
[infinitepushbackup]
# Whether to enable automatic backups. If this option is True then a backup
# process will be started after every mercurial command that modifies the
# repo, for example, commit, amend, histedit, rebase etc.
autobackup = False
# path to the directory where pushback logs should be stored
logdir = path/to/dir
# Backup at most maxheadstobackup heads, other heads are ignored.
# Negative number means backup everything.
maxheadstobackup = -1
# Nodes that should not be backed up. Ancestors of these nodes won't be
# backed up either
dontbackupnodes = []
# Special option that may be used to trigger re-backuping. For example,
# if there was a bug in infinitepush backups, then changing the value of
# this option will force all clients to make a "clean" backup
backupgeneration = 0
# Hostname value to use. If not specified then socket.gethostname() will
# be used
hostname = ''
# Enable reporting of infinitepush backup status as a summary at the end
# of smartlog.
enablestatus = False
# Whether or not to save information about the latest successful backup.
# This information includes the local revision number and unix timestamp
# of the last time we successfully made a backup.
savelatestbackupinfo = False
"""
from __future__ import absolute_import
import collections
import errno
import json
import os
import re
import socket
import stat
import subprocess
import time
from mercurial.node import (
bin,
hex,
nullrev,
short,
)
from mercurial.i18n import _
from mercurial import (
bundle2,
changegroup,
commands,
discovery,
dispatch,
encoding,
error,
extensions,
hg,
localrepo,
lock as lockmod,
phases,
policy,
registrar,
scmutil,
util,
)
from . import bundleparts
getscratchbookmarkspart = bundleparts.getscratchbookmarkspart
getscratchbranchparts = bundleparts.getscratchbranchparts
from hgext3rd import shareutil
osutil = policy.importmod(r'osutil')
cmdtable = {}
command = registrar.command(cmdtable)
revsetpredicate = registrar.revsetpredicate()
templatekeyword = registrar.templatekeyword()
backupbookmarktuple = collections.namedtuple('backupbookmarktuple',
['hostname', 'reporoot', 'localbookmark'])
class backupstate(object):
def __init__(self):
self.heads = set()
self.localbookmarks = {}
def empty(self):
return not self.heads and not self.localbookmarks
class WrongPermissionsException(Exception):
def __init__(self, logdir):
self.logdir = logdir
restoreoptions = [
('', 'reporoot', '', 'root of the repo to restore'),
('', 'user', '', 'user who ran the backup'),
('', 'hostname', '', 'hostname of the repo to restore'),
]
_backuplockname = 'infinitepushbackup.lock'
def extsetup(ui):
if ui.configbool('infinitepushbackup', 'autobackup', False):
extensions.wrapfunction(dispatch, 'runcommand',
_autobackupruncommandwrapper)
extensions.wrapfunction(localrepo.localrepository, 'transaction',
_transaction)
@command('pushbackup',
[('', 'background', None, 'run backup in background')])
def backup(ui, repo, dest=None, **opts):
"""
Pushes commits, bookmarks and heads to infinitepush.
New non-extinct commits are saved since the last `hg pushbackup`
or since 0 revision if this backup is the first.
Local bookmarks are saved remotely as:
infinitepush/backups/USERNAME/HOST/REPOROOT/bookmarks/LOCAL_BOOKMARK
Local heads are saved remotely as:
infinitepush/backups/USERNAME/HOST/REPOROOT/heads/HEAD_HASH
"""
if opts.get('background'):
_dobackgroundbackup(ui, repo, dest)
return 0
try:
# Wait at most 30 seconds, because that's the average backup time
timeout = 30
srcrepo = shareutil.getsrcrepo(repo)
with lockmod.lock(srcrepo.vfs, _backuplockname, timeout=timeout):
return _dobackup(ui, repo, dest, **opts)
except error.LockHeld as e:
if e.errno == errno.ETIMEDOUT:
ui.warn(_('timeout waiting on backup lock\n'))
return 0
else:
raise
@command('pullbackup', restoreoptions)
def restore(ui, repo, dest=None, **opts):
"""
Pulls commits from infinitepush that were previously saved with
`hg pushbackup`.
If user has only one backup for the `dest` repo then it will be restored.
But user may have backed up many local repos that points to `dest` repo.
These local repos may reside on different hosts or in different
repo roots. It makes restore ambiguous; `--reporoot` and `--hostname`
options are used to disambiguate.
"""
other = _getremote(repo, ui, dest, **opts)
sourcereporoot = opts.get('reporoot')
sourcehostname = opts.get('hostname')
namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))
allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,
sourcehostname, namingmgr)
if len(allbackupstates) == 0:
ui.warn(_('no backups found!'))
return 1
_checkbackupstates(allbackupstates)
__, backupstate = allbackupstates.popitem()
pullcmd, pullopts = _getcommandandoptions('^pull')
# pull backuped heads and nodes that are pointed by bookmarks
pullopts['rev'] = list(backupstate.heads |
set(backupstate.localbookmarks.values()))
if dest:
pullopts['source'] = dest
result = pullcmd(ui, repo, **pullopts)
with repo.wlock(), repo.lock(), repo.transaction('bookmark') as tr:
changes = []
for book, hexnode in backupstate.localbookmarks.iteritems():
if hexnode in repo:
changes.append((book, bin(hexnode)))
else:
ui.warn(_('%s not found, not creating %s bookmark') %
(hexnode, book))
repo._bookmarks.applychanges(repo, tr, changes)
# manually write local backup state and flag to not autobackup
# just after we restored, which would be pointless
_writelocalbackupstate(repo.vfs,
list(backupstate.heads),
backupstate.localbookmarks)
repo.ignoreautobackup = True
return result
@command('getavailablebackups',
[('', 'user', '', _('username, defaults to current user')),
('', 'json', None, _('print available backups in json format'))])
def getavailablebackups(ui, repo, dest=None, **opts):
other = _getremote(repo, ui, dest, **opts)
sourcereporoot = opts.get('reporoot')
sourcehostname = opts.get('hostname')
namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))
allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,
sourcehostname, namingmgr)
if opts.get('json'):
jsondict = collections.defaultdict(list)
for hostname, reporoot in allbackupstates.keys():
jsondict[hostname].append(reporoot)
# make sure the output is sorted. That's not an efficient way to
# keep list sorted but we don't have that many backups.
jsondict[hostname].sort()
ui.write('%s\n' % json.dumps(jsondict))
else:
if not allbackupstates:
ui.write(_('no backups available for %s\n') % namingmgr.username)
ui.write(_('user %s has %d available backups:\n') %
(namingmgr.username, len(allbackupstates)))
for hostname, reporoot in sorted(allbackupstates.keys()):
ui.write(_('%s on %s\n') % (reporoot, hostname))
@command('debugcheckbackup',
[('', 'all', None, _('check all backups that user have')),
] + restoreoptions)
def checkbackup(ui, repo, dest=None, **opts):
"""
Checks that all the nodes that backup needs are available in bundlestore
This command can check either specific backup (see restoreoptions) or all
backups for the user
"""
sourcereporoot = opts.get('reporoot')
sourcehostname = opts.get('hostname')
other = _getremote(repo, ui, dest, **opts)
namingmgr = BackupBookmarkNamingManager(ui, repo, opts.get('user'))
allbackupstates = _downloadbackupstate(ui, other, sourcereporoot,
sourcehostname, namingmgr)
if not opts.get('all'):
_checkbackupstates(allbackupstates)
ret = 0
while allbackupstates:
key, bkpstate = allbackupstates.popitem()
ui.status(_('checking %s on %s\n') % (key[1], key[0]))
if not _dobackupcheck(bkpstate, ui, repo, dest, **opts):
ret = 255
return ret
@command('debugwaitbackup', [('', 'timeout', '', 'timeout value')])
def waitbackup(ui, repo, timeout):
try:
if timeout:
timeout = int(timeout)
else:
timeout = -1
except ValueError:
raise error.Abort('timeout should be integer')
try:
repo = shareutil.getsrcrepo(repo)
with lockmod.lock(repo.vfs, _backuplockname, timeout=timeout):
pass
except error.LockHeld as e:
if e.errno == errno.ETIMEDOUT:
raise error.Abort(_('timeout while waiting for backup'))
raise
@command('isbackedup',
[('r', 'rev', [], _('show the specified revision or revset'), _('REV'))])
def isbackedup(ui, repo, **opts):
"""checks if commit was backed up to infinitepush
If no revision are specified then it checks working copy parent
"""
revs = opts.get('rev')
if not revs:
revs = ['.']
bkpstate = _readlocalbackupstate(ui, repo)
unfi = repo.unfiltered()
backeduprevs = unfi.revs('draft() and ::%ls', bkpstate.heads)
for r in scmutil.revrange(unfi, revs):
ui.write(_(unfi[r].hex() + ' '))
ui.write(_('backed up' if r in backeduprevs else 'not backed up'))
ui.write(_('\n'))
@revsetpredicate('backedup')
def backedup(repo, subset, x):
"""Draft changesets that have been backed up by infinitepush"""
unfi = repo.unfiltered()
bkpstate = _readlocalbackupstate(repo.ui, repo)
return subset & unfi.revs('draft() and ::%ls and not hidden()',
bkpstate.heads)
@revsetpredicate('notbackedup')
def notbackedup(repo, subset, x):
"""Changesets that have not yet been backed up by infinitepush"""
bkpstate = _readlocalbackupstate(repo.ui, repo)
bkpheads = set(bkpstate.heads)
candidates = set(_backupheads(repo.ui, repo))
notbackeduprevs = set()
# Find all revisions that are ancestors of the expected backup heads,
# stopping when we reach either a public commit or a known backup head.
while candidates:
candidate = candidates.pop()
if candidate not in bkpheads:
ctx = repo[candidate]
rev = ctx.rev()
if rev not in notbackeduprevs and ctx.phase() != phases.public:
# This rev may not have been backed up. Record it, and add its
# parents as candidates.
notbackeduprevs.add(rev)
candidates.update([p.hex() for p in ctx.parents()])
if notbackeduprevs:
# Some revisions in this set may actually have been backed up by
# virtue of being an ancestor of a different backup head, which may
# have been hidden since the backup was made. Find these and remove
# them from the set.
unfi = repo.unfiltered()
candidates = bkpheads
while candidates:
candidate = candidates.pop()
if candidate in unfi:
ctx = unfi[candidate]
if ctx.phase() != phases.public:
notbackeduprevs.discard(ctx.rev())
candidates.update([p.hex() for p in ctx.parents()])
return subset & notbackeduprevs
@templatekeyword('backingup')
def backingup(repo, ctx, **args):
"""Whether infinitepush is currently backing up commits."""
# If the backup lock exists then a backup should be in progress.
srcrepo = shareutil.getsrcrepo(repo)
return srcrepo.vfs.lexists(_backuplockname)
def smartlogsummary(ui, repo):
if not ui.configbool('infinitepushbackup', 'enablestatus'):
return
# Don't output the summary if a backup is currently in progress.
srcrepo = shareutil.getsrcrepo(repo)
if srcrepo.vfs.lexists(_backuplockname):
return
unbackeduprevs = repo.revs('notbackedup()')
# Count the number of changesets that haven't been backed up for 10 minutes.
# If there is only one, also print out its hash.
backuptime = time.time() - 10 * 60 # 10 minutes ago
count = 0
singleunbackeduprev = None
for rev in unbackeduprevs:
if repo[rev].date()[0] <= backuptime:
singleunbackeduprev = rev
count += 1
if count > 0:
if count > 1:
ui.warn(_('note: %d changesets are not backed up.\n') % count)
else:
ui.warn(_('note: changeset %s is not backed up.\n') %
short(repo[singleunbackeduprev].node()))
ui.warn(_('Run `hg pushbackup` to perform a backup. If this fails,\n'
'please report to the Source Control @ FB group.\n'))
def _autobackupruncommandwrapper(orig, lui, repo, cmd, fullargs, *args):
'''
If this wrapper is enabled then auto backup is started after every command
that modifies a repository.
Since we don't want to start auto backup after read-only commands,
then this wrapper checks if this command opened at least one transaction.
If yes then background backup will be started.
'''
# For chg, do not wrap the "serve" runcommand call
if 'CHGINTERNALMARK' in encoding.environ:
return orig(lui, repo, cmd, fullargs, *args)
try:
return orig(lui, repo, cmd, fullargs, *args)
finally:
if getattr(repo, 'txnwasopened', False) \
and not getattr(repo, 'ignoreautobackup', False):
lui.debug("starting infinitepush autobackup in the background\n")
_dobackgroundbackup(lui, repo)
def _transaction(orig, self, *args, **kwargs):
''' Wrapper that records if a transaction was opened.
If a transaction was opened then we want to start background backup process.
This hook records the fact that transaction was opened.
'''
self.txnwasopened = True
return orig(self, *args, **kwargs)
def _backupheads(ui, repo):
"""Returns the set of heads that should be backed up in this repo."""
maxheadstobackup = ui.configint('infinitepushbackup',
'maxheadstobackup', -1)
revset = 'heads(draft()) & not obsolete()'
backupheads = [ctx.hex() for ctx in repo.set(revset)]
if maxheadstobackup > 0:
backupheads = backupheads[-maxheadstobackup:]
elif maxheadstobackup == 0:
backupheads = []
return set(backupheads)
def _dobackup(ui, repo, dest, **opts):
ui.status(_('starting backup %s\n') % time.strftime('%H:%M:%S %d %b %Y %Z'))
start = time.time()
# to handle multiple working copies correctly
repo = shareutil.getsrcrepo(repo)
currentbkpgenerationvalue = _readbackupgenerationfile(repo.vfs)
newbkpgenerationvalue = ui.configint('infinitepushbackup',
'backupgeneration', 0)
if currentbkpgenerationvalue != newbkpgenerationvalue:
# Unlinking local backup state will trigger re-backuping
_deletebackupstate(repo)
_writebackupgenerationfile(repo.vfs, newbkpgenerationvalue)
bkpstate = _readlocalbackupstate(ui, repo)
# this variable stores the local store info (tip numeric revision and date)
# which we use to quickly tell if our backup is stale
afterbackupinfo = _getlocalinfo(repo)
# This variable will store what heads will be saved in backup state file
# if backup finishes successfully
afterbackupheads = _backupheads(ui, repo)
other = _getremote(repo, ui, dest, **opts)
outgoing, badhexnodes = _getrevstobackup(repo, ui, other,
afterbackupheads - bkpstate.heads)
# If remotefilelog extension is enabled then there can be nodes that we
# can't backup. In this case let's remove them from afterbackupheads
afterbackupheads.difference_update(badhexnodes)
# As afterbackupheads this variable stores what heads will be saved in
# backup state file if backup finishes successfully
afterbackuplocalbooks = _getlocalbookmarks(repo)
afterbackuplocalbooks = _filterbookmarks(
afterbackuplocalbooks, repo, afterbackupheads)
newheads = afterbackupheads - bkpstate.heads
removedheads = bkpstate.heads - afterbackupheads
newbookmarks = _dictdiff(afterbackuplocalbooks, bkpstate.localbookmarks)
removedbookmarks = _dictdiff(bkpstate.localbookmarks, afterbackuplocalbooks)
namingmgr = BackupBookmarkNamingManager(ui, repo)
bookmarkstobackup = _getbookmarkstobackup(
repo, newbookmarks, removedbookmarks,
newheads, removedheads, namingmgr)
# Special case if backup state is empty. Clean all backup bookmarks from the
# server.
if bkpstate.empty():
bookmarkstobackup[namingmgr.getbackupheadprefix()] = ''
bookmarkstobackup[namingmgr.getbackupbookmarkprefix()] = ''
# Wrap deltaparent function to make sure that bundle takes less space
# See _deltaparent comments for details
extensions.wrapfunction(changegroup.cg2packer, 'deltaparent', _deltaparent)
try:
bundler = _createbundler(ui, repo, other)
bundler.addparam("infinitepush", "True")
backup = False
if outgoing and outgoing.missing:
backup = True
parts = getscratchbranchparts(repo, other, outgoing,
confignonforwardmove=False,
ui=ui, bookmark=None,
create=False)
for part in parts:
bundler.addpart(part)
if bookmarkstobackup:
backup = True
bundler.addpart(getscratchbookmarkspart(other, bookmarkstobackup))
if backup:
_sendbundle(bundler, other)
_writelocalbackupstate(repo.vfs, afterbackupheads,
afterbackuplocalbooks)
if ui.config('infinitepushbackup', 'savelatestbackupinfo'):
_writelocalbackupinfo(repo.vfs, **afterbackupinfo)
else:
ui.status(_('nothing to backup\n'))
finally:
# cleanup ensures that all pipes are flushed
cleanup = getattr(other, '_cleanup', None) or getattr(other, 'cleanup')
try:
cleanup()
except Exception:
ui.warn(_('remote connection cleanup failed\n'))
ui.status(_('finished in %f seconds\n') % (time.time() - start))
extensions.unwrapfunction(changegroup.cg2packer, 'deltaparent',
_deltaparent)
return 0
def _dobackgroundbackup(ui, repo, dest=None):
background_cmd = ['hg', 'pushbackup']
if dest:
background_cmd.append(dest)
logfile = None
logdir = ui.config('infinitepushbackup', 'logdir')
if logdir:
# make newly created files and dirs non-writable
oldumask = os.umask(0o022)
try:
try:
username = util.shortuser(ui.username())
except Exception:
username = 'unknown'
if not _checkcommonlogdir(logdir):
raise WrongPermissionsException(logdir)
userlogdir = os.path.join(logdir, username)
util.makedirs(userlogdir)
if not _checkuserlogdir(userlogdir):
raise WrongPermissionsException(userlogdir)
reporoot = repo.origroot
reponame = os.path.basename(reporoot)
_removeoldlogfiles(userlogdir, reponame)
logfile = _getlogfilename(logdir, username, reponame)
except (OSError, IOError) as e:
ui.debug('infinitepush backup log is disabled: %s\n' % e)
except WrongPermissionsException as e:
ui.debug(('%s directory has incorrect permission, ' +
'infinitepush backup logging will be disabled\n') %
e.logdir)
finally:
os.umask(oldumask)
if not logfile:
logfile = os.devnull
with open(logfile, 'a') as f:
subprocess.Popen(background_cmd, shell=False, stdout=f,
stderr=subprocess.STDOUT)
def _dobackupcheck(bkpstate, ui, repo, dest, **opts):
remotehexnodes = sorted(
set(bkpstate.heads).union(bkpstate.localbookmarks.values()))
if not remotehexnodes:
return True
other = _getremote(repo, ui, dest, **opts)
batch = other.iterbatch()
for hexnode in remotehexnodes:
batch.lookup(hexnode)
batch.submit()
lookupresults = batch.results()
i = 0
try:
for i, r in enumerate(lookupresults):
# iterate over results to make it throw if revision
# was not found
pass
return True
except error.RepoError:
ui.warn(_('unknown revision %r\n') % remotehexnodes[i])
return False
_backuplatestinfofile = 'infinitepushlatestbackupinfo'
_backupstatefile = 'infinitepushbackupstate'
_backupgenerationfile = 'infinitepushbackupgeneration'
# Common helper functions
def _getlocalinfo(repo):
localinfo = {}
localinfo['rev'] = repo[repo.changelog.tip()].rev()
localinfo['time'] = int(time.time())
return localinfo
def _getlocalbookmarks(repo):
localbookmarks = {}
for bookmark, node in repo._bookmarks.iteritems():
hexnode = hex(node)
localbookmarks[bookmark] = hexnode
return localbookmarks
def _filterbookmarks(localbookmarks, repo, headstobackup):
'''Filters out some bookmarks from being backed up
Filters out bookmarks that do not point to ancestors of headstobackup or
public commits
'''
headrevstobackup = [repo[hexhead].rev() for hexhead in headstobackup]
ancestors = repo.changelog.ancestors(headrevstobackup, inclusive=True)
filteredbooks = {}
for bookmark, hexnode in localbookmarks.iteritems():
if (repo[hexnode].rev() in ancestors or
repo[hexnode].phase() == phases.public):
filteredbooks[bookmark] = hexnode
return filteredbooks
def _downloadbackupstate(ui, other, sourcereporoot, sourcehostname, namingmgr):
pattern = namingmgr.getcommonuserprefix()
fetchedbookmarks = other.listkeyspatterns('bookmarks', patterns=[pattern])
allbackupstates = collections.defaultdict(backupstate)
for book, hexnode in fetchedbookmarks.iteritems():
parsed = _parsebackupbookmark(book, namingmgr)
if parsed:
if sourcereporoot and sourcereporoot != parsed.reporoot:
continue
if sourcehostname and sourcehostname != parsed.hostname:
continue
key = (parsed.hostname, parsed.reporoot)
if parsed.localbookmark:
bookname = parsed.localbookmark
allbackupstates[key].localbookmarks[bookname] = hexnode
else:
allbackupstates[key].heads.add(hexnode)
else:
ui.warn(_('wrong format of backup bookmark: %s') % book)
return allbackupstates
def _checkbackupstates(allbackupstates):
if len(allbackupstates) == 0:
raise error.Abort('no backups found!')
hostnames = set(key[0] for key in allbackupstates.iterkeys())
reporoots = set(key[1] for key in allbackupstates.iterkeys())
if len(hostnames) > 1:
raise error.Abort(
_('ambiguous hostname to restore: %s') % sorted(hostnames),
hint=_('set --hostname to disambiguate'))
if len(reporoots) > 1:
raise error.Abort(
_('ambiguous repo root to restore: %s') % sorted(reporoots),
hint=_('set --reporoot to disambiguate'))
class BackupBookmarkNamingManager(object):
def __init__(self, ui, repo, username=None):
self.ui = ui
self.repo = repo
if not username:
username = util.shortuser(ui.username())
self.username = username
self.hostname = self.ui.config('infinitepushbackup', 'hostname')
if not self.hostname:
self.hostname = socket.gethostname()
def getcommonuserprefix(self):
return '/'.join((self._getcommonuserprefix(), '*'))
def getcommonprefix(self):
return '/'.join((self._getcommonprefix(), '*'))
def getbackupbookmarkprefix(self):
return '/'.join((self._getbackupbookmarkprefix(), '*'))
def getbackupbookmarkname(self, bookmark):
bookmark = _escapebookmark(bookmark)
return '/'.join((self._getbackupbookmarkprefix(), bookmark))
def getbackupheadprefix(self):
return '/'.join((self._getbackupheadprefix(), '*'))
def getbackupheadname(self, hexhead):
return '/'.join((self._getbackupheadprefix(), hexhead))
def _getbackupbookmarkprefix(self):
return '/'.join((self._getcommonprefix(), 'bookmarks'))
def _getbackupheadprefix(self):
return '/'.join((self._getcommonprefix(), 'heads'))
def _getcommonuserprefix(self):
return '/'.join(('infinitepush', 'backups', self.username))
def _getcommonprefix(self):
reporoot = self.repo.origroot
result = '/'.join((self._getcommonuserprefix(), self.hostname))
if not reporoot.startswith('/'):
result += '/'
result += reporoot
if result.endswith('/'):
result = result[:-1]
return result
def _escapebookmark(bookmark):
'''
If `bookmark` contains "bookmarks" as a substring then replace it with
"bookmarksbookmarks". This will make parsing remote bookmark name
unambigious.
'''
bookmark = encoding.fromlocal(bookmark)
return bookmark.replace('bookmarks', 'bookmarksbookmarks')
def _unescapebookmark(bookmark):
bookmark = encoding.tolocal(bookmark)
return bookmark.replace('bookmarksbookmarks', 'bookmarks')
def _getremote(repo, ui, dest, **opts):
path = ui.paths.getpath(dest, default=('infinitepush', 'default'))
if not path:
raise error.Abort(_('default repository not configured!'),
hint=_("see 'hg help config.paths'"))
dest = path.pushloc or path.loc
return hg.peer(repo, opts, dest)
def _getcommandandoptions(command):
cmd = commands.table[command][0]
opts = dict(opt[1:3] for opt in commands.table[command][1])
return cmd, opts
# Backup helper functions
def _deltaparent(orig, self, revlog, rev, p1, p2, prev):
# This version of deltaparent prefers p1 over prev to use less space
dp = revlog.deltaparent(rev)
if dp == nullrev and not revlog.storedeltachains:
# send full snapshot only if revlog configured to do so
return nullrev
return p1
def _getbookmarkstobackup(repo, newbookmarks, removedbookmarks,
newheads, removedheads, namingmgr):
bookmarkstobackup = {}
for bookmark, hexnode in removedbookmarks.items():
backupbookmark = namingmgr.getbackupbookmarkname(bookmark)
bookmarkstobackup[backupbookmark] = ''
for bookmark, hexnode in newbookmarks.items():
backupbookmark = namingmgr.getbackupbookmarkname(bookmark)
bookmarkstobackup[backupbookmark] = hexnode
for hexhead in removedheads:
headbookmarksname = namingmgr.getbackupheadname(hexhead)
bookmarkstobackup[headbookmarksname] = ''
for hexhead in newheads:
headbookmarksname = namingmgr.getbackupheadname(hexhead)
bookmarkstobackup[headbookmarksname] = hexhead
return bookmarkstobackup
def _createbundler(ui, repo, other):
bundler = bundle2.bundle20(ui, bundle2.bundle2caps(other))
# Disallow pushback because we want to avoid taking repo locks.
# And we don't need pushback anyway
capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo,
allowpushback=False))
bundler.newpart('replycaps', data=capsblob)
return bundler
def _sendbundle(bundler, other):
stream = util.chunkbuffer(bundler.getchunks())
try:
other.unbundle(stream, ['force'], other.url())
except error.BundleValueError as exc:
raise error.Abort(_('missing support for %s') % exc)
def findcommonoutgoing(repo, ui, other, heads):
if heads:
# Avoid using remotenames fastheaddiscovery heuristic. It uses
# remotenames file to quickly find commonoutgoing set, but it can
# result in sending public commits to infinitepush servers.
# For example:
#
# o draft
# /
# o C1
# |
# ...
# |
# o remote/master
#
# pushbackup in that case results in sending to the infinitepush server
# all public commits from 'remote/master' to C1. It increases size of
# the bundle + it may result in storing data about public commits
# in infinitepush table.
with ui.configoverride({("remotenames", "fastheaddiscovery"): False}):
nodes = map(repo.changelog.node, heads)
return discovery.findcommonoutgoing(repo, other, onlyheads=nodes)
else:
return None
def _getrevstobackup(repo, ui, other, headstobackup):
# In rare cases it's possible to have a local node without filelogs.
# This is possible if remotefilelog is enabled and if the node was
# stripped server-side. We want to filter out these bad nodes and all
# of their descendants.
badnodes = ui.configlist('infinitepushbackup', 'dontbackupnodes', [])
badnodes = [node for node in badnodes if node in repo]
badrevs = [repo[node].rev() for node in badnodes]
badnodesdescendants = repo.set('%ld::', badrevs) if badrevs else set()
badnodesdescendants = set(ctx.hex() for ctx in badnodesdescendants)
filteredheads = filter(lambda head: head in badnodesdescendants,
headstobackup)
if filteredheads:
ui.warn(_('filtering nodes: %s\n') % filteredheads)
ui.log('infinitepushbackup', 'corrupted nodes found',
infinitepushbackupcorruptednodes='failure')
headstobackup = filter(lambda head: head not in badnodesdescendants,
headstobackup)
revs = list(repo[hexnode].rev() for hexnode in headstobackup)
outgoing = findcommonoutgoing(repo, ui, other, revs)
nodeslimit = 1000
if outgoing and len(outgoing.missing) > nodeslimit:
# trying to push too many nodes usually means that there is a bug
# somewhere. Let's be safe and avoid pushing too many nodes at once
raise error.Abort('trying to back up too many nodes: %d' %
(len(outgoing.missing),))
return outgoing, set(filteredheads)
def _localbackupstateexists(repo):
return repo.vfs.exists(_backupstatefile)
def _deletebackupstate(repo):
return repo.vfs.tryunlink(_backupstatefile)
def _readlocalbackupstate(ui, repo):
repo = shareutil.getsrcrepo(repo)
if not _localbackupstateexists(repo):
return backupstate()
with repo.vfs(_backupstatefile) as f:
try:
state = json.loads(f.read())
if (not isinstance(state['bookmarks'], dict) or
not isinstance(state['heads'], list)):
raise ValueError('bad types of bookmarks or heads')
result = backupstate()
result.heads = set(map(str, state['heads']))
result.localbookmarks = state['bookmarks']
return result
except (ValueError, KeyError, TypeError) as e:
ui.warn(_('corrupt file: %s (%s)\n') % (_backupstatefile, e))
return backupstate()
return backupstate()
def _writelocalbackupstate(vfs, heads, bookmarks):
with vfs(_backupstatefile, 'w') as f:
f.write(json.dumps({'heads': list(heads), 'bookmarks': bookmarks}))
def _readbackupgenerationfile(vfs):
try:
with vfs(_backupgenerationfile) as f:
return int(f.read())
except (IOError, OSError, ValueError):
return 0
def _writebackupgenerationfile(vfs, backupgenerationvalue):
with vfs(_backupgenerationfile, 'w', atomictemp=True) as f:
f.write(str(backupgenerationvalue))
def _writelocalbackupinfo(vfs, rev, time):
with vfs(_backuplatestinfofile, 'w', atomictemp=True) as f:
f.write(('backuprevision=%d\nbackuptime=%d\n') % (rev, time))
# Restore helper functions
def _parsebackupbookmark(backupbookmark, namingmgr):
'''Parses backup bookmark and returns info about it
Backup bookmark may represent either a local bookmark or a head.
Returns None if backup bookmark has wrong format or tuple.
First entry is a hostname where this bookmark came from.
Second entry is a root of the repo where this bookmark came from.
Third entry in a tuple is local bookmark if backup bookmark
represents a local bookmark and None otherwise.
'''
backupbookmarkprefix = namingmgr._getcommonuserprefix()
commonre = '^{0}/([-\w.]+)(/.*)'.format(re.escape(backupbookmarkprefix))
bookmarkre = commonre + '/bookmarks/(.*)$'
headsre = commonre + '/heads/[a-f0-9]{40}$'
match = re.search(bookmarkre, backupbookmark)
if not match:
match = re.search(headsre, backupbookmark)
if not match:
return None
# It's a local head not a local bookmark.
# That's why localbookmark is None
return backupbookmarktuple(hostname=match.group(1),
reporoot=match.group(2),
localbookmark=None)
return backupbookmarktuple(hostname=match.group(1),
reporoot=match.group(2),
localbookmark=_unescapebookmark(match.group(3)))
_timeformat = '%Y%m%d'
def _getlogfilename(logdir, username, reponame):
'''Returns name of the log file for particular user and repo
Different users have different directories inside logdir. Log filename
consists of reponame (basename of repo path) and current day
(see _timeformat). That means that two different repos with the same name
can share the same log file. This is not a big problem so we ignore it.
'''
currentday = time.strftime(_timeformat)
return os.path.join(logdir, username, reponame + currentday)
def _removeoldlogfiles(userlogdir, reponame):
existinglogfiles = []
for entry in osutil.listdir(userlogdir):
filename = entry[0]
fullpath = os.path.join(userlogdir, filename)
if filename.startswith(reponame) and os.path.isfile(fullpath):
try:
time.strptime(filename[len(reponame):], _timeformat)
except ValueError:
continue
existinglogfiles.append(filename)
# _timeformat gives us a property that if we sort log file names in
# descending order then newer files are going to be in the beginning
existinglogfiles = sorted(existinglogfiles, reverse=True)
# Delete logs that are older than 5 days
maxlogfilenumber = 5
if len(existinglogfiles) > maxlogfilenumber:
for filename in existinglogfiles[maxlogfilenumber:]:
os.unlink(os.path.join(userlogdir, filename))
def _checkcommonlogdir(logdir):
'''Checks permissions of the log directory
We want log directory to actually be a directory, have restricting
deletion flag set (sticky bit)
'''
try:
st = os.stat(logdir)
return stat.S_ISDIR(st.st_mode) and st.st_mode & stat.S_ISVTX
except OSError:
# is raised by os.stat()
return False
def _checkuserlogdir(userlogdir):
'''Checks permissions of the user log directory
We want user log directory to be writable only by the user who created it
and be owned by `username`
'''
try:
st = os.stat(userlogdir)
# Check that `userlogdir` is owned by `username`
if os.getuid() != st.st_uid:
return False
return ((st.st_mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)) ==
stat.S_IWUSR)
except OSError:
# is raised by os.stat()
return False
def _dictdiff(first, second):
'''Returns new dict that contains items from the first dict that are missing
from the second dict.
'''
result = {}
for book, hexnode in first.items():
if second.get(book) != hexnode:
result[book] = hexnode
return result

hgext/infinitepush/infinitepushcommands.py

hg, hg,
patch, patch,
registrar, registrar,
scmutil, scmutil,
util, util,
) )
from . import ( from . import (
backupcommands,
common, common,
) )
downloadbundle = common.downloadbundle downloadbundle = common.downloadbundle
cmdtable = backupcommands.cmdtable cmdtable = {}
command = registrar.command(cmdtable) command = registrar.command(cmdtable)
@command('debugfillinfinitepushmetadata', @command('debugfillinfinitepushmetadata',
[('', 'node', [], 'node to fill metadata for')]) [('', 'node', [], 'node to fill metadata for')])
def debugfillinfinitepushmetadata(ui, repo, **opts): def debugfillinfinitepushmetadata(ui, repo, **opts):
'''Special command that fills infinitepush metadata for a node '''Special command that fills infinitepush metadata for a node
''' '''
hgext/infinitepush/__init__.py