diff --git a/hgext3rd/fastannotate/context.py b/hgext3rd/fastannotate/context.py
--- a/hgext3rd/fastannotate/context.py
+++ b/hgext3rd/fastannotate/context.py
@@ -97,8 +97,22 @@
         ctx = _revsingle(repo, rev)
     else:
         ctx = repo[rev]
-    fctx = ctx[path]
-    if adjustctx is not None:
+
+    # If we don't need to adjust the linkrev, create the filectx using the
+    # changectx instead of using ctx[path]. This means it already has the
+    # changectx information, so blame -u will be able to look directly at the
+    # commitctx object instead of having to resolve it by going through the
+    # manifest. In a lazy-manifest world this can prevent us from downloading a
+    # lot of data.
+    if adjustctx is None:
+        # ctx.rev() is None means it's the working copy, which is a special
+        # case.
+        if ctx.rev() is None:
+            fctx = ctx[path]
+        else:
+            fctx = repo.filectx(path, changeid=ctx.rev())
+    else:
+        fctx = ctx[path]
         if adjustctx == 'linkrev':
             introrev = fctx.linkrev()
         else:
diff --git a/tests/test-treemanifest-blame.t b/tests/test-treemanifest-blame.t
new file mode 100644
--- /dev/null
+++ b/tests/test-treemanifest-blame.t
@@ -0,0 +1,69 @@
+  $ . "$TESTDIR/library.sh"
+
+  $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
+  $ export PYTHONPATH
+  $ cat >> $HGRCPATH <<EOF
+  > [treemanifest]
+  > sendtrees=True
+  > [fastannotate]
+  > mainbranch=default
+  > modes=fctx
+  > EOF
+
+Setup the server
+
+  $ hginit master
+  $ cd master
+  $ cat >> .hg/hgrc <<EOF
+  > [extensions]
+  > fastannotate=$TESTDIR/../hgext3rd/fastannotate
+  > treemanifest=$TESTDIR/../treemanifest
+  > [treemanifest]
+  > server=True
+  > [remotefilelog]
+  > server=True
+  > shallowtrees=True
+  > [fastannotate]
+  > server=True
+  > EOF
+
+Make local commits on the server
+  $ mkdir subdir
+  $ echo x >> subdir/x
+  $ hg commit -qAm 'add subdir/x'
+  $ echo x >> subdir/x
+  $ hg commit -qAm 'modify subdir/x'
+  $ echo x >> subdir/x
+  $ hg commit -qAm 'modify subdir/x 2'
+  $ echo x >> subdir/x
+  $ hg commit -qAm 'modify subdir/x 3'
+
+Run blame on client
+  $ cd ..
+  $ hgcloneshallow ssh://user@dummy/master client -q
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
+  $ cd client
+  $ cat >> .hg/hgrc <<EOF
+  > [extensions]
+  > fastannotate=$TESTDIR/../hgext3rd/fastannotate
+  > fastmanifest=$TESTDIR/../fastmanifest
+  > treemanifest=$TESTDIR/../treemanifest
+  > [fastmanifest]
+  > usecache=False
+  > usetree=True
+  > [treemanifest]
+  > demanddownload=True
+  > [fastannotate]
+  > clientfetchthreshold=2
+  > EOF
+  $ clearcache
+  $ hg prefetch -r 'tip^::tip'
+  4 trees fetched over * (glob)
+  2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
+
+- Verify no trees are downloaded
+  $ hg blame -r tip -u subdir/x --pager=off
+  test: x
+  test: x
+  test: x
+  test: x