filereader cleanup (#30191)

* filereader cleanup

* make name return url

* remove unused imports
old-commit-hash: 86b90a8ba263df067406e0089eb507c3a397640e

tools/lib/filereader.py

@@ -3,9 +3,13 @@ from openpilot.tools.lib.url_file import URLFile
 
 DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/")
 
-def FileReader(fn, debug=False):
+def resolve_name(fn):
   if fn.startswith("cd:/"):
-    fn = fn.replace("cd:/", DATA_ENDPOINT)
+    return fn.replace("cd:/", DATA_ENDPOINT)
+  return fn
+
+def FileReader(fn, debug=False):
+  fn = resolve_name(fn)
   if fn.startswith(("http://", "https://")):
     return URLFile(fn, debug=debug)
   return open(fn, "rb")

tools/lib/framereader.py

@@ -17,7 +17,7 @@ from openpilot.tools.lib.exceptions import DataUnreadableError
 from openpilot.tools.lib.vidindex import hevc_index
 from openpilot.common.file_helpers import atomic_write_in_dir
 
-from openpilot.tools.lib.filereader import FileReader
+from openpilot.tools.lib.filereader import FileReader, resolve_name
 
 HEVC_SLICE_B = 0
 HEVC_SLICE_P = 1
@@ -60,6 +60,7 @@ def fingerprint_video(fn):
 
 
 def ffprobe(fn, fmt=None):
+  fn = resolve_name(fn)
   cmd = ["ffprobe",
          "-v", "quiet",
          "-print_format", "json",

tools/lib/url_file.py

@@ -1,8 +1,6 @@
 import os
 import time
-import tempfile
 import threading
-import urllib.parse
 import pycurl
 from hashlib import sha256
 from io import BytesIO
@@ -37,7 +35,8 @@ class URLFile:
       self._curl = self._tlocal.curl
     except AttributeError:
       self._curl = self._tlocal.curl = pycurl.Curl()
-    mkdirs_exists_ok(Paths.download_cache_root())
+    if not self._force_download:
+      mkdirs_exists_ok(Paths.download_cache_root())
 
   def __enter__(self):
     return self
@@ -65,12 +64,13 @@ class URLFile:
   def get_length(self):
     if self._length is not None:
       return self._length
+
     file_length_path = os.path.join(Paths.download_cache_root(), hash_256(self._url) + "_length")
-    if os.path.exists(file_length_path) and not self._force_download:
+    if not self._force_download and os.path.exists(file_length_path):
       with open(file_length_path) as file_length:
-          content = file_length.read()
-          self._length = int(content)
-          return self._length
+        content = file_length.read()
+        self._length = int(content)
+        return self._length
 
     self._length = self.get_length_online()
     if not self._force_download and self._length != -1:
@@ -173,24 +173,4 @@ class URLFile:
 
   @property
   def name(self):
-    """Returns a local path to file with the URLFile's contents.
-
-       This can be used to interface with modules that require local files.
-    """
-    if self._local_file is None:
-      _, ext = os.path.splitext(urllib.parse.urlparse(self._url).path)
-      local_fd, local_path = tempfile.mkstemp(suffix=ext)
-      try:
-        os.write(local_fd, self.read())
-        local_file = open(local_path, "rb")
-      except Exception:
-        os.remove(local_path)
-        raise
-      finally:
-        os.close(local_fd)
-
-      self._local_file = local_file
-      self.read = self._local_file.read
-      self.seek = self._local_file.seek
-
-    return self._local_file.name
+    return self._url