Currently, due to how urllib3 works, every file is loaded into memory before it is sent over the wire. Thus, with big files you'll end up like this:
Traceback (most recent call last):
File "/usr/local/bin/jottacloudclientscanner.py", line 67, in saferun
return apply(cmd, args)
File "/usr/local/lib/python2.7/dist-packages/jottacloudclient/jottacloud.py", line 87, in new
_new = JFS.up(jottapath, lf)
File "/usr/local/lib/python2.7/dist-packages/jottalib/JFS.py", line 638, in up
return self.post(url, None, files=files, params=params, extra_headers=headers)
File "/usr/local/lib/python2.7/dist-packages/jottalib/JFS.py", line 587, in post
r = self.session.post(url, data=content, params=params, files=files, headers=headers)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 424, in post
return self.request('POST', url, data=data, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests_cache/core.py", line 110, in request
hooks, stream, verify, cert)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 348, in request
prep = self.prepare_request(req)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 286, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 289, in prepare
self.prepare_body(data, files)
File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 426, in prepare_body
(body, content_type) = self._encode_files(files, data)
File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 144, in _encode_files
body, content_type = encode_multipart_formdata(new_fields)
File "/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/filepost.py", line 101, in encode_multipart_formdata
return body.getvalue(), content_type
MemoryError
havardgulldahl
commented
9 years ago
Currently, due to how urllib3 works, every file is loaded into memory before it is sent over the wire. Thus, with big files you'll end up like this:
Traceback (most recent call last): File "/usr/local/bin/jottacloudclientscanner.py", line 67, in saferun return apply(cmd, args) File "/usr/local/lib/python2.7/dist-packages/jottacloudclient/jottacloud.py", line 87, in new _new = JFS.up(jottapath, lf) File "/usr/local/lib/python2.7/dist-packages/jottalib/JFS.py", line 638, in up return self.post(url, None, files=files, params=params, extra_headers=headers) File "/usr/local/lib/python2.7/dist-packages/jottalib/JFS.py", line 587, in post r = self.session.post(url, data=content, params=params, files=files, headers=headers) File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 424, in post return self.request('POST', url, data=data, **kwargs) File "/usr/local/lib/python2.7/dist-packages/requests_cache/core.py", line 110, in request hooks, stream, verify, cert) File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 348, in request prep = self.prepare_request(req) File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 286, in prepare_request hooks=merge_hooks(request.hooks, self.hooks), File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 289, in prepare self.prepare_body(data, files) File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 426, in prepare_body (body, content_type) = self._encode_files(files, data) File "/usr/local/lib/python2.7/dist-packages/requests/models.py", line 144, in _encode_files body, content_type = encode_multipart_formdata(new_fields) File "/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/filepost.py", line 101, in encode_multipart_formdata return body.getvalue(), content_type MemoryError
We'll either have to wait until urllib3 gets chunked uploads working, or add support for partial uploads in jottalib.