diff --git a/remotefilelog/__init__.py b/remotefilelog/__init__.py
--- a/remotefilelog/__init__.py
+++ b/remotefilelog/__init__.py
@@ -895,7 +895,9 @@
     ], _('hg debugkeepset'))
 def debugkeepset(ui, repo, **opts):
     # The command is used to measure keepset computation time
-    repackmod.keepset(repo)
+    def keyfn(fname, fnode):
+        return fileserverclient.getcachekey(repo.name, fname, hex(fnode))
+    repackmod.keepset(repo, keyfn)
     return
 
 @command('debugwaitonrepack', [
diff --git a/remotefilelog/repack.py b/remotefilelog/repack.py
--- a/remotefilelog/repack.py
+++ b/remotefilelog/repack.py
@@ -337,8 +337,16 @@
         processed.add(r)
 
         # populate keepkeys with keys from the current manifest
-        for filename, filenode in m.iteritems():
-            keepkeys.add(keyfn(filename, filenode))
+        if type(m) is dict:
+            # m is a result of diff of two manifests and is a dictionary that
+            # maps filename to ((newnode, newflag), (oldnode, oldflag)) tuple
+            for filename, diff in m.iteritems():
+                if diff[0][0] is not None:
+                    keepkeys.add(keyfn(filename, diff[0][0]))
+        else:
+            # m is a manifest object
+            for filename, filenode in m.iteritems():
+                keepkeys.add(keyfn(filename, filenode))
 
     return keepkeys
 
diff --git a/tests/test-remotefilelog-keepset.t b/tests/test-remotefilelog-keepset.t
new file mode 100644
--- /dev/null
+++ b/tests/test-remotefilelog-keepset.t
@@ -0,0 +1,40 @@
+  $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
+  $ export PYTHONPATH
+
+  $ . "$TESTDIR/library.sh"
+
+  $ hginit master
+  $ cd master
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > server=True
+  > serverexpiration=-1
+  > EOF
+  $ echo x > x
+  $ hg commit -qAm x
+  $ echo y > y
+  $ hg commit -qAm y
+  $ echo z > z
+  $ hg commit -qAm z
+  $ cd ..
+
+  $ hgcloneshallow ssh://user@dummy/master shallow -q
+  3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
+
+# Compute keepset for 0th and 2nd commit, which implies that we do not process
+# the 1st commit, therefore we diff 2nd manifest with the 0th manifest and
+# populate the keepkeys from the diff
+  $ cd shallow
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > pullprefetch=0+2
+  > EOF
+  $ hg debugkeepset
+
+# Compute keepset for all commits, which implies that we only process deltas of
+# manifests of commits 1 and 2 and therefore populate the keepkeys from deltas
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > pullprefetch=all()
+  > EOF
+  $ hg debugkeepset