fup retry

mattermost-dl.py

@@ -26,6 +26,18 @@ THROTTLE_DEFAULTS = {
     "file_delay_multiplier": 1.5,
 }
 
+RETRY_DEFAULTS = {
+    "max_attempts": 5,
+    "base_delay": 2.0,
+    "max_delay": 60.0,
+}
+
+RETRYABLE_REQUEST_ERRORS = (
+    requests.exceptions.ConnectionError,
+    requests.exceptions.Timeout,
+    requests.exceptions.ChunkedEncodingError,
+)
+
 
 class RequestThrottler:
     """Paces API calls with jitter and occasional longer pauses."""
@@ -56,6 +68,21 @@ class RequestThrottler:
         self._last_request_at = time.monotonic()
 
 
+def _parse_retry_config(config: dict) -> Optional[dict]:
+    retry = config.get("retry")
+    if retry is False:
+        return None
+    if retry is None or retry is True:
+        return dict(RETRY_DEFAULTS)
+    if isinstance(retry, dict):
+        if not retry.get("enabled", True):
+            return None
+        opts = dict(RETRY_DEFAULTS)
+        opts.update({k: v for k, v in retry.items() if k != "enabled"})
+        return opts
+    return None
+
+
 def _parse_throttle_config(config: dict) -> Optional[dict]:
     throttle = config.get("throttle")
     if not throttle:
@@ -72,24 +99,52 @@ def _parse_throttle_config(config: dict) -> Optional[dict]:
 
 
 def apply_throttle(d: Driver, config: dict) -> None:
-    opts = _parse_throttle_config(config)
+    throttle_opts = _parse_throttle_config(config)
-    if not opts:
+    retry_opts = _parse_retry_config(config)
+    if not throttle_opts and not retry_opts:
         return
 
-    throttler = RequestThrottler(**opts)
+    throttler = RequestThrottler(**(throttle_opts or THROTTLE_DEFAULTS))
     client = d.client
     original_make_request = client.make_request
 
     def throttled_make_request(method, endpoint, options=None, params=None, data=None, files=None, basepath=None):
-        throttler.wait(endpoint)
+        if throttle_opts:
-        return original_make_request(method, endpoint, options, params, data, files, basepath)
+            throttler.wait(endpoint)
+
+        attempts = retry_opts["max_attempts"] if retry_opts else 1
+        last_error = None
+        for attempt in range(attempts):
+            try:
+                return original_make_request(method, endpoint, options, params, data, files, basepath)
+            except RETRYABLE_REQUEST_ERRORS as error:
+                last_error = error
+                if attempt + 1 >= attempts:
+                    break
+                delay = min(
+                    retry_opts["base_delay"] * (2 ** attempt),
+                    retry_opts["max_delay"],
+                ) + random.uniform(0, 1)
+                print(
+                    f"Request failed ({error}), retrying in {delay:.1f}s "
+                    f"({attempt + 2}/{attempts})..."
+                )
+                time.sleep(delay)
+        raise last_error
 
     client.make_request = throttled_make_request
-    print(
+    if throttle_opts:
-        "Request throttling enabled "
+        print(
-        f"(delay {opts['min_delay']}-{opts['max_delay']}s, "
+            "Request throttling enabled "
-        f"pause every {opts['burst_pause_every']} requests)"
+            f"(delay {throttle_opts['min_delay']}-{throttle_opts['max_delay']}s, "
-    )
+            f"pause every {throttle_opts['burst_pause_every']} requests)"
+        )
+    if retry_opts:
+        print(
+            "Request retries enabled "
+            f"(up to {retry_opts['max_attempts']} attempts, "
+            f"backoff from {retry_opts['base_delay']}s)"
+        )
 
 
 def connect(host: str, login_token: str = None, username: str = None, password: str = None) -> Driver:
@@ -364,6 +419,15 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out
     # Create output directory
     output_base = pathlib.Path(output_base) / channel_name
     output_base.mkdir(parents=True, exist_ok=True)
+
+    existing_post_ids = set()
+    if merge and output_filepath.exists():
+        with open(output_filepath, encoding='utf8') as f:
+            existing_export = json.load(f)
+        existing_post_ids = {post["id"] for post in existing_export.get("posts", [])}
+        if existing_post_ids:
+            print(f"  skipping {len(existing_post_ids)} posts already exported")
+
     # Simplify all posts to contain only username, date, message and files in chronological order
     simple_posts = []
     for i_post, post in enumerate(reversed(all_posts)):
@@ -373,6 +437,9 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out
         if (before_ts and created > before_ts) or (after_ts and created <= after_ts):
             continue
 
+        if post["id"] in existing_post_ids:
+            continue
+
         user_id = post["user_id"]
         if user_id not in user_id_to_name:
             user_id_to_name[user_id] = d.users.get_user(user_id)["username"]
@@ -402,20 +469,24 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out
             for file in post["metadata"]["files"]:
                 if download_files:
                     filename = "%03d" % i_post + "_" + file["name"]
-                    print("Downloading", file["name"])
+                    dest = output_base / filename
-                    try:
+                    if merge and dest.exists():
-                        resp = d.files.get_file(file["id"])
+                        print(f"Skipping already downloaded {file['name']}")
-                    except ResourceNotFound:
-                        print(f"File not found, skipping: {file['name']}")
-                        filenames.append(file["name"])
-                        continue
-                    # Mattermost Driver unfortunately parses json files to dicts
-                    if isinstance(resp, dict):
-                        with open(output_base / filename, "w") as f:
-                            json.dump(resp, f)
                     else:
-                        with open(output_base / filename, "wb") as f:
+                        print("Downloading", file["name"])
-                            f.write(resp.content)
+                        try:
+                            resp = d.files.get_file(file["id"])
+                        except ResourceNotFound:
+                            print(f"File not found, skipping: {file['name']}")
+                            filenames.append(file["name"])
+                            continue
+                        # Mattermost Driver unfortunately parses json files to dicts
+                        if isinstance(resp, dict):
+                            with open(dest, "w") as f:
+                                json.dump(resp, f)
+                        else:
+                            with open(dest, "wb") as f:
+                                f.write(resp.content)
 
                 filenames.append(file["name"])
             simple_post["files"] = filenames