tools: add retry mechanism for API requests (#1480)

tools/lib/api.py

@@ -1,5 +1,6 @@
 import os
 import requests
+from requests.adapters import HTTPAdapter, Retry
 API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
 
 # TODO: this should be merged into common.api
@@ -11,6 +12,9 @@ class CommaApi:
     if token:
       self.session.headers['Authorization'] = 'JWT ' + token
 
+    retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
+    self.session.mount('https://', HTTPAdapter(max_retries=retries))
+
   def request(self, method, endpoint, **kwargs):
     with self.session.request(method, API_HOST + '/' + endpoint, **kwargs) as resp:
       resp_json = resp.json()