dependency: remove pycurl package (#30771)

* Update qcomgpsd.py

* Update url_file.py

* remove pycurl

* requests -> urllib3

* unused

* redundant

* fix import

* Revert "requests -> urllib3"

This reverts commit 7ca39e618917986f50356519d2795a432e3a5280.

* headless

* fix trail

* use requests.exceptions.RequestException

* use fp.tell

* fix indents

* reorder imports

* change timeout

* fix debug timing

* remove exception

* add timeout

* missing headers

* move to constructor

* move import

* unused import

* fix debug

* try

* no retries

poetry.lock

@@ -3206,16 +3206,6 @@ files = [
     {file = "pycryptodome-3.19.0.tar.gz", hash = "sha256:bc35d463222cdb4dbebd35e0784155c81e161b9284e567e7e933d722e533331e"},
 ]
 
-[[package]]
-name = "pycurl"
-version = "7.45.2"
-description = "PycURL -- A Python Interface To The cURL library"
-optional = false
-python-versions = ">=3.5"
-files = [
-    {file = "pycurl-7.45.2.tar.gz", hash = "sha256:5730590be0271364a5bddd9e245c9cc0fb710c4cbacbdd95264a3122d23224ca"},
-]
-
 [[package]]
 name = "pyee"
 version = "11.1.0"
@@ -4655,17 +4645,6 @@ notebook = ["ipywidgets (>=6)"]
 slack = ["slack-sdk"]
 telegram = ["requests"]
 
-[[package]]
-name = "types-pycurl"
-version = "7.45.2.5"
-description = "Typing stubs for pycurl"
-optional = false
-python-versions = "*"
-files = [
-    {file = "types-pycurl-7.45.2.5.tar.gz", hash = "sha256:cc07e4b1e388b98e1a935bc12ef33eec8e7294496d0a81ed50c46a31bf5df5ee"},
-    {file = "types_pycurl-7.45.2.5-py3-none-any.whl", hash = "sha256:e433f13a266245be770018eabc6a50139ea26150f75df5b0ee698d291575582b"},
-]
-
 [[package]]
 name = "types-requests"
 version = "2.31.0.10"
@@ -4902,4 +4881,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "~3.11"
-content-hash = "eeb5efb6161eaec95fb863ddc9ec08afb540ed3cd5d521646a6a759ddf33ab2e"
+content-hash = "26e963a3e9236b0e29466a0bca243215395a780c88485e9c17f6518969334890"

pyproject.toml

@@ -110,7 +110,6 @@ polyline = "*"
 # these should be removed
 markdown-it-py = "*"
 timezonefinder = "*"
-pycurl = "*"
 setproctitle = "*"
 
 
@@ -153,7 +152,6 @@ sphinx-rtd-theme = "*"
 sphinx-sitemap = "*"
 tabulate = "*"
 tenacity = "*"
-types-pycurl = "*"
 types-requests = "*"
 types-tabulate = "*"
 tqdm = "*"

system/qcomgpsd/qcomgpsd.py

@@ -5,7 +5,7 @@ import signal
 import itertools
 import math
 import time
-import pycurl
+import requests
 import shutil
 import subprocess
 import datetime
@@ -102,27 +102,18 @@ def gps_enabled() -> bool:
 
 def download_assistance():
   try:
-    c = pycurl.Curl()
-    c.setopt(pycurl.URL, ASSISTANCE_URL)
-    c.setopt(pycurl.NOBODY, 1)
-    c.setopt(pycurl.CONNECTTIMEOUT, 2)
-    c.perform()
-    bytes_n = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
-    c.close()
-    if bytes_n > 1e5:
-      cloudlog.error("Qcom assistance data larger than expected")
-      return
+    response = requests.get(ASSISTANCE_URL, timeout=5, stream=True)
 
     with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp:
-      c = pycurl.Curl()
-      c.setopt(pycurl.URL, ASSISTANCE_URL)
-      c.setopt(pycurl.CONNECTTIMEOUT, 5)
+      for chunk in response.iter_content(chunk_size=8192):
+        fp.write(chunk)
+        if fp.tell() > 1e5:
+          cloudlog.error("Qcom assistance data larger than expected")
+          return
 
-      c.setopt(pycurl.WRITEDATA, fp)
-      c.perform()
-      c.close()
-      os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE)
-  except pycurl.error:
+    os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE)
+
+  except requests.exceptions.RequestException:
     cloudlog.exception("Failed to download assistance file")
     return
 

tools/lib/url_file.py

@@ -1,10 +1,11 @@
 import os
 import time
 import threading
-import pycurl
 from hashlib import sha256
-from io import BytesIO
+from urllib3 import PoolManager, Retry
+from urllib3.util import Timeout
 from tenacity import retry, wait_random_exponential, stop_after_attempt
+
 from openpilot.common.file_helpers import atomic_write_in_dir
 from openpilot.system.hardware.hw import Paths
 #  Cache chunk size
@@ -35,13 +36,14 @@ class URLFile:
     if cache is not None:
       self._force_download = not cache
 
-    try:
-      self._curl = self._tlocal.curl
-    except AttributeError:
-      self._curl = self._tlocal.curl = pycurl.Curl()
     if not self._force_download:
       os.makedirs(Paths.download_cache_root(), exist_ok=True)
 
+    try:
+      self._http_client = URLFile._tlocal.http_client
+    except AttributeError:
+      self._http_client = URLFile._tlocal.http_client = PoolManager()
+
   def __enter__(self):
     return self
 
@@ -53,17 +55,10 @@ class URLFile:
 
   @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
   def get_length_online(self):
-    c = self._curl
-    c.reset()
-    c.setopt(pycurl.NOSIGNAL, 1)
-    c.setopt(pycurl.TIMEOUT_MS, 500000)
-    c.setopt(pycurl.FOLLOWLOCATION, True)
-    c.setopt(pycurl.URL, self._url)
-    c.setopt(c.NOBODY, 1)
-    c.perform()
-    length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD))
-    c.reset()
-    return length
+    timeout = Timeout(connect=50.0, read=500.0)
+    response = self._http_client.request('HEAD', self._url, timeout=timeout, preload_content=False)
+    length = response.headers.get('content-length', 0)
+    return int(length)
 
   def get_length(self):
     if self._length is not None:
@@ -117,7 +112,7 @@ class URLFile:
   @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
   def read_aux(self, ll=None):
     download_range = False
-    headers = ["Connection: keep-alive"]
+    headers = {'Connection': 'keep-alive'}
     if self._pos != 0 or ll is not None:
       if ll is None:
         end = self.get_length() - 1
@@ -125,50 +120,29 @@ class URLFile:
         end = min(self._pos + ll, self.get_length()) - 1
       if self._pos >= end:
         return b""
-      headers.append(f"Range: bytes={self._pos}-{end}")
+      headers['Range'] = f"bytes={self._pos}-{end}"
       download_range = True
 
-    dats = BytesIO()
-    c = self._curl
-    c.setopt(pycurl.URL, self._url)
-    c.setopt(pycurl.WRITEDATA, dats)
-    c.setopt(pycurl.NOSIGNAL, 1)
-    c.setopt(pycurl.TIMEOUT_MS, 500000)
-    c.setopt(pycurl.HTTPHEADER, headers)
-    c.setopt(pycurl.FOLLOWLOCATION, True)
-
     if self._debug:
-      print("downloading", self._url)
-
-      def header(x):
-        if b'MISS' in x:
-          print(x.strip())
-
-      c.setopt(pycurl.HEADERFUNCTION, header)
-
-      def test(debug_type, debug_msg):
-       print("  debug(%d): %s" % (debug_type, debug_msg.strip()))
-
-      c.setopt(pycurl.VERBOSE, 1)
-      c.setopt(pycurl.DEBUGFUNCTION, test)
       t1 = time.time()
 
-    c.perform()
+    timeout = Timeout(connect=50.0, read=500.0)
+    response = self._http_client.request('GET', self._url, timeout=timeout, preload_content=False, headers=headers)
+    ret = response.data
 
     if self._debug:
       t2 = time.time()
       if t2 - t1 > 0.1:
-        print(f"get {self._url} {headers!r} {t2 - t1:.f} slow")
+        print(f"get {self._url} {headers!r} {t2 - t1:.3f} slow")
 
-    response_code = c.getinfo(pycurl.RESPONSE_CODE)
+    response_code = response.status
     if response_code == 416:  # Requested Range Not Satisfiable
-      raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+      raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(ret)[:500]}")
     if download_range and response_code != 206:  # Partial Content
-      raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+      raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(ret)[:500]}")
     if (not download_range) and response_code != 200:  # OK
-      raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+      raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(ret)[:500]}")
 
-    ret = dats.getvalue()
     self._pos += len(ret)
     return ret