21541: Fix KeyError, segfaults, and memory use issues

[arvados.git] / sdk / python / arvados / http_to_keep.py

diff --git a/sdk/python/arvados/http_to_keep.py b/sdk/python/arvados/http_to_keep.py
index b37ab5910914b67454038de18682b1bc41b5863f..f247afeaffb13071dde5b750bfd16417417ce8ed 100644 (file)
--- a/sdk/python/arvados/http_to_keep.py
+++ b/sdk/python/arvados/http_to_keep.py
@@ -2,23 +2,20 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from __future__ import division
-from future import standard_library
-standard_library.install_aliases()
-
-import email.utils
-import time
+import calendar
+import dataclasses
 import datetime
-import re
-import arvados
-import arvados.collection
-import urllib.parse
+import email.utils
 import logging
-import calendar
+import re
+import time
+import typing
 import urllib.parse
+
 import pycurl
-import dataclasses
-import typing
+
+import arvados
+import arvados.collection
 from arvados._pycurlhelper import PyCurlHelper
 
 logger = logging.getLogger('arvados.http_import')
@@ -182,6 +179,10 @@ class _Downloader(PyCurlHelper):
         mt = re.match(r'^HTTP\/(\d(\.\d)?) ([1-5]\d\d) ([^\r\n\x00-\x08\x0b\x0c\x0e-\x1f\x7f]*)\r\n$', self._headers["x-status-line"])
         code = int(mt.group(3))
 
+        if not self.name:
+            logger.error("Cannot determine filename from URL or headers")
+            return
+
         if code == 200:
             self.target = self.collection.open(self.name, "wb")
 
@@ -193,7 +194,10 @@ class _Downloader(PyCurlHelper):
         self.count += len(chunk)
 
         if self.target is None:
-            return
+            # "If this number is not equal to the size of the byte
+            # string, this signifies an error and libcurl will abort
+            # the request."
+            return 0
 
         self.target.write(chunk)
         loopnow = time.time()