[~/hg-committed/tests] ./run-tests.py -l test-lfs-test-server.t

--- /home/wlis/hg-committed/tests/test-lfs-test-server.t
+++ /home/wlis/hg-committed/tests/test-lfs-test-server.t.err
@@ -43,54 +43,98 @@
   pushing to ../repo2
   searching for changes
   lfs: uploading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
-  1 changesets found
-  uncompressed size of bundle content:
-       * (changelog) (glob)
-       * (manifests) (glob)
-       *  a (glob)
-  adding changesets
-  adding manifests
-  adding file changes
-  added 1 changesets with 1 changes to 1 files
+  lfs: failed: TypeError("object of type 'file' has no len()",) (remaining retry 5)
+  lfs: failed: TypeError("object of type 'file' has no len()",) (remaining retry 4)
+  lfs: failed: TypeError("object of type 'file' has no len()",) (remaining retry 3)
+  lfs: failed: TypeError("object of type 'file' has no len()",) (remaining retry 2)
+  lfs: failed: TypeError("object of type 'file' has no len()",) (remaining retry 1)
+  ** unknown exception encountered, please report by visiting
+  ** https://mercurial-scm.org/wiki/BugTracker
+  ** Python 2.7.5 (default, Aug  4 2017, 00:39:18) [GCC 4.8.5 20150623 (Red Hat 4.8.5-16)]
+  ** Mercurial Distributed SCM (version 4.4.1+203-4da86512789c)
+  ** Extensions loaded: lfs
+  Traceback (most recent call last):
+    File "/home/wlis/hg-committed/hg", line 41, in <module>
+      dispatch.run()
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 88, in run
+      status = (dispatch(req) or 0) & 255
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 177, in dispatch
+      ret = _runcatch(req)
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 318, in _runcatch
+      return _callcatch(ui, _runcatchfunc)
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 326, in _callcatch
+      return scmutil.callcatch(ui, func)
+    File "/home/wlis/hg-committed/mercurial/scmutil.py", line 154, in callcatch
+      return func()
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 308, in _runcatchfunc
+      return _dispatch(req)
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 912, in _dispatch
+      cmdpats, cmdoptions)
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 667, in runcommand
+      ret = _runcommand(ui, options, cmd, d)
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 920, in _runcommand
+      return cmdfunc()
+    File "/home/wlis/hg-committed/mercurial/dispatch.py", line 909, in <lambda>
+      d = lambda: util.checksignature(func)(ui, *args, **strcmdopt)
+    File "/home/wlis/hg-committed/mercurial/util.py", line 1188, in check
+      return func(*args, **kwargs)
+    File "/home/wlis/hg-committed/mercurial/commands.py", line 4160, in push
+      opargs=opargs)
+    File "/home/wlis/hg-committed/mercurial/exchange.py", line 475, in push
+      _pushbundle2(pushop)
+    File "/home/wlis/hg-committed/mercurial/exchange.py", line 1023, in _pushbundle2
+      ret = partgen(pushop, bundler)
+    File "/home/wlis/hg-committed/mercurial/exchange.py", line 797, in _pushb2ctx
+      pushop.repo.prepushoutgoinghooks(pushop)
+    File "/home/wlis/hg-committed/mercurial/util.py", line 3125, in __call__
+      results.append(hook(*args))
+    File "/home/wlis/hg-committed/hgext/lfs/wrapper.py", line 263, in prepush
+      return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
+    File "/home/wlis/hg-committed/hgext/lfs/wrapper.py", line 254, in uploadblobsfromrevs
+      uploadblobs(repo, pointers)
+    File "/home/wlis/hg-committed/hgext/lfs/wrapper.py", line 306, in uploadblobs
+      remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
+    File "/home/wlis/hg-committed/hgext/lfs/blobstore.py", line 133, in writebatch
+      self._batch(pointers, fromstore, 'upload')
+    File "/home/wlis/hg-committed/hgext/lfs/blobstore.py", line 294, in _batch
+      for _one, oid in oids:
+    File "/home/wlis/hg-committed/hgext/lfs/blobstore.py", line 278, in transfer
+      self._basictransfer(obj, action, localstore)
+    File "/home/wlis/hg-committed/hgext/lfs/blobstore.py", line 235, in _basictransfer
+      req = self.urlopener.open(request)
+    File "/usr/lib64/python2.7/urllib2.py", line 429, in open
+      req = meth(req)
+    File "/usr/lib64/python2.7/urllib2.py", line 1152, in do_request_
+      'Content-length', '%d' % len(data))
+  TypeError: object of type 'file' has no len()
+  [1]

 Clear the cache to force a download
   $ rm -rf `hg config lfs.usercache`
   $ cd ../repo2
   $ hg update tip -v
-  resolving manifests
-  getting a
-  lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
-  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved

 When the server has some blobs already

   $ hg mv a b
+  a: $ENOENT$
+  abort: no files to copy
+  [255]
   $ echo ANOTHER-LARGE-FILE > c
   $ echo ANOTHER-LARGE-FILE2 > d
   $ hg commit -m b-and-c -A b c d
+  b: $ENOENT$
+  abort: failed to mark all new/missing files as added/removed
+  [255]
   $ hg push ../repo1 -v | grep -v '^  '
   pushing to ../repo1
-  searching for changes
-  lfs: need to transfer 2 objects (39 bytes)
-  lfs: uploading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
-  lfs: uploading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
-  1 changesets found
-  uncompressed size of bundle content:
-  adding changesets
-  adding manifests
-  adding file changes
-  added 1 changesets with 3 changes to 3 files
+  no changes found

 Clear the cache to force a download
   $ rm -rf `hg config lfs.usercache`
   $ hg --repo ../repo1 update tip -v
-  resolving manifests
-  getting b
-  getting c
-  lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
-  getting d
-  lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
-  3 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved

 Check error message when the remote missed a blob:


ERROR: test-lfs-test-server.t output changed
!
Failed test-lfs-test-server.t: output changed
# Ran 1 tests, 0 skipped, 1 failed.
python hash seed: 1273209229
[~/hg-committed/tests]

# HG changeset patch
# User Matt Harbison <matt_harbison@yahoo.com>
# Date 1515094298 18000
#      Thu Jan 04 14:31:38 2018 -0500
# Branch bzr_convert
# Node ID b0abd77f295edbf1df58674bc1ef7a6bc57e4096
# Parent  0653b44c70685f9ed6f5cf1689ca08f7bd2dbe34
xxx-lfs: attempt to isolate upload corruption with workers

(Please don't queue this- it's only to help Wojciech isolate this.)

It doesn't look like all of the data is being sent over the socket, although it
appears that all of the data is being read from the file.  Here's a sample
transcript of an upload, with the subsequent wall of html removed.  In this
case, all 4 files failed checksum verification on the server side.  The size in
the "Sent %d bytes of data" line corresponds to what the server ends up with,
and the other sizes match what the client starts with.

I have no idea why this seems to work when 'lfs.url' is http instead of https.

$ hg push
...
pushing to lighttpd
searching for changes
12f746f3f2493b2f39da7ecf63d7ee19c6ac9ec6a4fcd8c229da8a522cb12765: size 5278176
932b4ee4def2b434f85435d9e3e19ca8ba99ce9a065a61524b429a9d5e9b2e9c: size 5258384
ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7: size 2062258
dbc0ae9f8b05a7f84770ea303db5c1601500295548b3253e51f8889fcb38cc0a: size 5103733
Broken pipe!
Broken pipe!
Broken pipe!
No more data on ccdf...bbc7.  Read 2062258, expected 2062258
Sent 939954 bytes of data
No more data on dbc0...cc0a.  Read 5103733, expected 5103733
Sent 4931701 bytes of data
No more data on 932b...2e9c.  Read 5258384, expected 5258384
Sent 5176464 bytes of data
No more data on 12f7...2765.  Read 5278176, expected 5278176
Sent 5138912 bytes of data
<html with java.net.SocketTimeoutException stacktrace removed>

diff --git a/hgext/lfs/blobstore.py b/hgext/lfs/blobstore.py
--- a/hgext/lfs/blobstore.py
+++ b/hgext/lfs/blobstore.py
@@ -64,12 +64,14 @@ class filewithprogress(object):
     Useful to provide progress information for how many bytes are read.
     """
 
-    def __init__(self, fp, callback):
+    def __init__(self, fp, callback, name):
         self._fp = fp
         self._callback = callback # func(readsize)
         fp.seek(0, os.SEEK_END)
         self._len = fp.tell()
+        self.total = 0
         fp.seek(0)
+        self.name = name
 
     def __len__(self):
         return self._len
@@ -79,9 +81,12 @@ class filewithprogress(object):
             return b''
         data = self._fp.read(size)
         if data:
+            self.total += len(data)
             if self._callback:
                 self._callback(len(data))
         else:
+            print('No more data on %s.  Read %d, expected %d' %
+                  (self.name, self.total, self._len))
             self._fp.close()
             self._fp = None
         return data
@@ -100,6 +105,12 @@ class local(object):
         self.cachevfs = lfsvfs(usercache)
         self.ui = repo.ui
 
+    def open(self, oid):
+        """Open a file descriptor for the named blob."""
+        if self.cachevfs.exists(oid):
+            return self.cachevfs(oid)
+        return self.vfs(oid)
+
     def write(self, oid, data, verify=True):
         """Write blob to local blobstore."""
         if verify:
@@ -254,9 +265,9 @@ class _gitlfsremote(object):
         request = util.urlreq.request(href)
         if action == 'upload':
             # If uploading blobs, read data from local blobstore.
-            with localstore.vfs(oid) as fp:
+            with localstore.open(oid) as fp:
                 _verifyfile(oid, fp)
-            request.data = filewithprogress(localstore.vfs(oid), None)
+            request.data = filewithprogress(localstore.open(oid), None, oid)
             request.get_method = lambda: 'PUT'
 
         for k, v in headers:
@@ -271,6 +282,8 @@ class _gitlfsremote(object):
                     break
                 response += data
         except util.urlerr.httperror as ex:
+            print('%s: %s' % (oid, ex.read()))
+            print('%s: %s' % (oid, ex.info()))
             raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
                                  % (ex, oid, action))
 
@@ -288,6 +301,7 @@ class _gitlfsremote(object):
         sizes = {}
         for obj in objects:
             sizes[obj.get('oid')] = obj.get('size', 0)
+            print('%s: size %d' % (obj.get('oid'), obj.get('size', 0)))
         topic = {'upload': _('lfs uploading'),
                  'download': _('lfs downloading')}[action]
         if len(objects) > 1:
diff --git a/mercurial/keepalive.py b/mercurial/keepalive.py
--- a/mercurial/keepalive.py
+++ b/mercurial/keepalive.py
@@ -553,14 +553,18 @@ def safesend(self, str):
             if self.debuglevel > 0:
                 print("sending a read()able")
             data = read(blocksize)
+            total = 0
             while data:
+                total += len(data)
                 self.sock.sendall(data)
                 data = read(blocksize)
+            print('Sent %d bytes of data' % total)
         else:
             self.sock.sendall(str)
     except socket.error as v:
         reraise = True
         if v[0] == errno.EPIPE:      # Broken pipe
+            print('Broken pipe!')
             if self._HTTPConnection__state == httplib._CS_REQ_SENT:
                 self._broken_pipe_resp = None
                 self._broken_pipe_resp = self.getresponse()