( )⚙ D125 phabricator: add a small language to query Differential Revisions

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D125

phabricator: add a small language to query Differential Revisions
ClosedPublic

Authored by quark on Jul 18 2017, 5:10 AM.

Details

Reviewers
durin42
Group Reviewers
hg-reviewers
Commits
rHG539541779010: phabricator: add a small language to query Differential Revisions
Summary

Previously, phabread can only be used to read a single patch, or a single
stack of patches. In the future, we want to have more complex queries like
filtering with status (open, accepted, closed, etc), or maybe more complex
like filtering by reviewers etc. The command line flag approach won't scale
with that.

Besides, we might want to have other commands to update Differential
Revision status in batch, like accepting a stack using a single command.

Therefore, this patch adds a small language. It has basic set operations:
&, +, - and an ancestor operator to support --stack.

Test Plan

Try querying this Phabricator instance:

hg phabread 1+2 # 1, 2
hg phabread D2+D1 # 2, 1
hg phabread ':118-115+:2-1' # 114, 116, 117, 118, 2
hg phabread '((:118-(D115+117)))&:117' # 114, 116
hg phabread ':2&:117' --debug # differential.query is called only once

Make sure the output is expected and prefetch works.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

quark created this revision.Jul 18 2017, 5:10 AM
quark abandoned this revision.Jul 18 2017, 5:37 PM
quark reclaimed this revision.Aug 1 2017, 4:39 PM
quark updated this revision to Diff 487.
quark updated this revision to Diff 555.Aug 4 2017, 3:55 PM
quark updated this revision to Diff 570.Aug 4 2017, 6:13 PM
quark updated this revision to Diff 743.Aug 10 2017, 2:59 PM
quark updated this revision to Diff 840.Aug 13 2017, 12:41 AM
durin42 accepted this revision.Aug 15 2017, 5:43 PM
This revision is now accepted and ready to land.Aug 15 2017, 5:43 PM
Closed by commit rHG539541779010: phabricator: add a small language to query Differential Revisions (authored by quark). · Explain WhyAug 15 2017, 5:49 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mcontrib/phabricator.py (178 lines)

Diff 961

contrib/phabricator.py

# Repo callsign. If a repo has a URL https://$HOST/diffusion/FOO, then its # Repo callsign. If a repo has a URL https://$HOST/diffusion/FOO, then its
# callsign is "FOO". # callsign is "FOO".
callsign = FOO callsign = FOO
""" """
from __future__ import absolute_import from __future__ import absolute_import
import itertools
import json import json
import operator
import re import re
from mercurial.node import bin, nullid from mercurial.node import bin, nullid
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
cmdutil, cmdutil,
context, context,
encoding, encoding,
error, error,
mdiff, mdiff,
obsutil, obsutil,
parser,
patch, patch,
registrar, registrar,
scmutil, scmutil,
smartset,
tags, tags,
url as urlmod, url as urlmod,
util, util,
) )
cmdtable = {} cmdtable = {}
command = registrar.command(cmdtable) command = registrar.command(cmdtable)
ui.write(('%s\n' % desc), label='phabsend.desc') ui.write(('%s\n' % desc), label='phabsend.desc')
if ui.promptchoice(_('Phabsend the above changes (yn)?' if ui.promptchoice(_('Phabsend the above changes (yn)?'
'$$ &Yes $$ &No')): '$$ &Yes $$ &No')):
return False return False
return True return True
def querydrev(repo, params, stack=False): # Small language to specify differential revisions. Support symbols: (), :X,
# +, and -.
_elements = {
# token-type: binding-strength, primary, prefix, infix, suffix
'(': (12, None, ('group', 1, ')'), None, None),
':': (8, None, ('ancestors', 8), None, None),
'&': (5, None, None, ('and_', 5), None),
'+': (4, None, None, ('add', 4), None),
'-': (4, None, None, ('sub', 4), None),
')': (0, None, None, None, None),
'symbol': (0, 'symbol', None, None, None),
'end': (0, None, None, None, None),
}
def _tokenize(text):
text = text.replace(' ', '') # remove space
view = memoryview(text) # zero-copy slice
special = '():+-&'
pos = 0
length = len(text)
while pos < length:
symbol = ''.join(itertools.takewhile(lambda ch: ch not in special,
view[pos:]))
if symbol:
yield ('symbol', symbol, pos)
pos += len(symbol)
else: # special char
yield (text[pos], None, pos)
pos += 1
yield ('end', None, pos)
def _parse(text):
tree, pos = parser.parser(_elements).parse(_tokenize(text))
if pos != len(text):
raise error.ParseError('invalid token', pos)
return tree
def _parsedrev(symbol):
"""str -> int or None, ex. 'D45' -> 45; '12' -> 12; 'x' -> None"""
if symbol.startswith('D') and symbol[1:].isdigit():
return int(symbol[1:])
if symbol.isdigit():
return int(symbol)
def _prefetchdrevs(tree):
"""return ({single-drev-id}, {ancestor-drev-id}) to prefetch"""
drevs = set()
ancestordrevs = set()
op = tree[0]
if op == 'symbol':
r = _parsedrev(tree[1])
if r:
drevs.add(r)
elif op == 'ancestors':
r, a = _prefetchdrevs(tree[1])
drevs.update(r)
ancestordrevs.update(r)
ancestordrevs.update(a)
else:
for t in tree[1:]:
r, a = _prefetchdrevs(t)
drevs.update(r)
ancestordrevs.update(a)
return drevs, ancestordrevs
def querydrev(repo, spec):
"""return a list of "Differential Revision" dicts """return a list of "Differential Revision" dicts
params is the input of "differential.query" API, and is expected to match spec is a string using a simple query language, see docstring in phabread
just a single Differential Revision. for details.
A "Differential Revision dict" looks like: A "Differential Revision dict" looks like:
{ {
"id": "2", "id": "2",
"phid": "PHID-DREV-672qvysjcczopag46qty", "phid": "PHID-DREV-672qvysjcczopag46qty",
"title": "example", "title": "example",
"uri": "https://phab.example.com/D2", "uri": "https://phab.example.com/D2",
"phabricator:projects": [], "phabricator:projects": [],
"phabricator:depends-on": [ "phabricator:depends-on": [
"PHID-DREV-gbapp366kutjebt7agcd" "PHID-DREV-gbapp366kutjebt7agcd"
] ]
}, },
"repositoryPHID": "PHID-REPO-hub2hx62ieuqeheznasv", "repositoryPHID": "PHID-REPO-hub2hx62ieuqeheznasv",
"sourcePath": null "sourcePath": null
} }
If stack is True, return a list of "Differential Revision dict"s in an
order that the latter ones depend on the former ones. Otherwise, return a
list of a unique "Differential Revision dict".
""" """
prefetched = {} # {id or phid: drev}
def fetch(params): def fetch(params):
"""params -> single drev or None""" """params -> single drev or None"""
key = (params.get(r'ids') or params.get(r'phids') or [None])[0] key = (params.get(r'ids') or params.get(r'phids') or [None])[0]
if key in prefetched: if key in prefetched:
return prefetched[key] return prefetched[key]
# Otherwise, send the request. If we're fetching a stack, be smarter drevs = callconduit(repo, 'differential.query', params)
# and fetch more ids in one batch, even if it could be unnecessary.
batchparams = params
if stack and len(params.get(r'ids', [])) == 1:
i = int(params[r'ids'][0])
# developer config: phabricator.batchsize
batchsize = repo.ui.configint('phabricator', 'batchsize', 12)
batchparams = {'ids': range(max(1, i - batchsize), i + 1)}
drevs = callconduit(repo, 'differential.query', batchparams)
# Fill prefetched with the result # Fill prefetched with the result
for drev in drevs: for drev in drevs:
prefetched[drev[r'phid']] = drev prefetched[drev[r'phid']] = drev
prefetched[int(drev[r'id'])] = drev prefetched[int(drev[r'id'])] = drev
if key not in prefetched: if key not in prefetched:
raise error.Abort(_('cannot get Differential Revision %r') % params) raise error.Abort(_('cannot get Differential Revision %r') % params)
return prefetched[key] return prefetched[key]
def getstack(topdrevids):
"""given a top, get a stack from the bottom, [id] -> [id]"""
visited = set() visited = set()
result = [] result = []
queue = [params] queue = [{r'ids': [i]} for i in topdrevids]
while queue: while queue:
params = queue.pop() params = queue.pop()
drev = fetch(params) drev = fetch(params)
if drev[r'id'] in visited: if drev[r'id'] in visited:
continue continue
visited.add(drev[r'id']) visited.add(drev[r'id'])
result.append(drev) result.append(int(drev[r'id']))
if stack:
auxiliary = drev.get(r'auxiliary', {}) auxiliary = drev.get(r'auxiliary', {})
depends = auxiliary.get(r'phabricator:depends-on', []) depends = auxiliary.get(r'phabricator:depends-on', [])
for phid in depends: for phid in depends:
queue.append({'phids': [phid]}) queue.append({'phids': [phid]})
result.reverse() result.reverse()
return result return smartset.baseset(result)
# Initialize prefetch cache
prefetched = {} # {id or phid: drev}
tree = _parse(spec)
drevs, ancestordrevs = _prefetchdrevs(tree)
# developer config: phabricator.batchsize
batchsize = repo.ui.configint('phabricator', 'batchsize', 12)
# Prefetch Differential Revisions in batch
tofetch = set(drevs)
for r in ancestordrevs:
tofetch.update(range(max(1, r - batchsize), r + 1))
if drevs:
fetch({r'ids': list(tofetch)})
getstack(list(ancestordrevs))
# Walk through the tree, return smartsets
def walk(tree):
op = tree[0]
if op == 'symbol':
drev = _parsedrev(tree[1])
if drev:
return smartset.baseset([drev])
else:
raise error.Abort(_('unknown symbol: %s') % tree[1])
elif op in {'and_', 'add', 'sub'}:
assert len(tree) == 3
return getattr(operator, op)(walk(tree[1]), walk(tree[2]))
elif op == 'group':
return walk(tree[1])
elif op == 'ancestors':
return getstack(walk(tree[1]))
else:
raise error.ProgrammingError('illegal tree: %r' % tree)
return [prefetched[r] for r in walk(tree)]
def getdescfromdrev(drev): def getdescfromdrev(drev):
"""get description (commit message) from "Differential Revision" """get description (commit message) from "Differential Revision"
This is similar to differential.getcommitmessage API. But we only care This is similar to differential.getcommitmessage API. But we only care
about limited fields: title, summary, test plan, and URL. about limited fields: title, summary, test plan, and URL.
""" """
title = drev[r'title'] title = drev[r'title']
if k in meta: if k in meta:
header += '# %s %s\n' % (_metanamemap[k], meta[k]) header += '# %s %s\n' % (_metanamemap[k], meta[k])
content = '%s%s\n%s' % (header, desc, body) content = '%s%s\n%s' % (header, desc, body)
write(encoding.unitolocal(content)) write(encoding.unitolocal(content))
@command('phabread', @command('phabread',
[('', 'stack', False, _('read dependencies'))], [('', 'stack', False, _('read dependencies'))],
_('REVID [OPTIONS]')) _('DREVSPEC [OPTIONS]'))
def phabread(ui, repo, revid, **opts): def phabread(ui, repo, spec, **opts):
"""print patches from Phabricator suitable for importing """print patches from Phabricator suitable for importing
REVID could be a Differential Revision identity, like ``D123``, or just the DREVSPEC could be a Differential Revision identity, like ``D123``, or just
number ``123``, or a full URL like ``https://phab.example.com/D123``. the number ``123``. It could also have common operators like ``+``, ``-``,
``&``, ``(``, ``)`` for complex queries. Prefix ``:`` could be used to
select a stack.
For example, ``:D6+8-(2+D4)`` selects a stack up to D6, plus D8 and exclude
D2 and D4.
If --stack is given, follow dependencies information and read all patches. If --stack is given, follow dependencies information and read all patches.
It is equivalent to the ``:`` operator.
""" """
try: if opts.get('stack'):
revid = int(revid.split('/')[-1].replace('D', '')) spec = ':(%s)' % spec
except ValueError: drevs = querydrev(repo, spec)
raise error.Abort(_('invalid Revision ID: %s') % revid)
drevs = querydrev(repo, {'ids': [revid]}, opts.get('stack'))
readpatch(repo, drevs, ui.write) readpatch(repo, drevs, ui.write)