From a41271724d13624b9eb9fed9c78d4b10bef6fcb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9C=D0=B0=D1=80?= =?UTF-8?q?=D0=B8=D0=BD=D0=BA=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 5 Jun 2026 15:46:14 +0700 Subject: [PATCH] fup retry --- mattermost-dl.py | 117 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 94 insertions(+), 23 deletions(-) diff --git a/mattermost-dl.py b/mattermost-dl.py index c75d39b..47c05fc 100644 --- a/mattermost-dl.py +++ b/mattermost-dl.py @@ -26,6 +26,18 @@ THROTTLE_DEFAULTS = { "file_delay_multiplier": 1.5, } +RETRY_DEFAULTS = { + "max_attempts": 5, + "base_delay": 2.0, + "max_delay": 60.0, +} + +RETRYABLE_REQUEST_ERRORS = ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, +) + class RequestThrottler: """Paces API calls with jitter and occasional longer pauses.""" @@ -56,6 +68,21 @@ class RequestThrottler: self._last_request_at = time.monotonic() +def _parse_retry_config(config: dict) -> Optional[dict]: + retry = config.get("retry") + if retry is False: + return None + if retry is None or retry is True: + return dict(RETRY_DEFAULTS) + if isinstance(retry, dict): + if not retry.get("enabled", True): + return None + opts = dict(RETRY_DEFAULTS) + opts.update({k: v for k, v in retry.items() if k != "enabled"}) + return opts + return None + + def _parse_throttle_config(config: dict) -> Optional[dict]: throttle = config.get("throttle") if not throttle: @@ -72,24 +99,52 @@ def _parse_throttle_config(config: dict) -> Optional[dict]: def apply_throttle(d: Driver, config: dict) -> None: - opts = _parse_throttle_config(config) - if not opts: + throttle_opts = _parse_throttle_config(config) + retry_opts = _parse_retry_config(config) + if not throttle_opts and not retry_opts: return - throttler = RequestThrottler(**opts) + throttler = RequestThrottler(**(throttle_opts or THROTTLE_DEFAULTS)) client = d.client original_make_request = client.make_request def throttled_make_request(method, endpoint, options=None, params=None, data=None, files=None, basepath=None): - throttler.wait(endpoint) - return original_make_request(method, endpoint, options, params, data, files, basepath) + if throttle_opts: + throttler.wait(endpoint) + + attempts = retry_opts["max_attempts"] if retry_opts else 1 + last_error = None + for attempt in range(attempts): + try: + return original_make_request(method, endpoint, options, params, data, files, basepath) + except RETRYABLE_REQUEST_ERRORS as error: + last_error = error + if attempt + 1 >= attempts: + break + delay = min( + retry_opts["base_delay"] * (2 ** attempt), + retry_opts["max_delay"], + ) + random.uniform(0, 1) + print( + f"Request failed ({error}), retrying in {delay:.1f}s " + f"({attempt + 2}/{attempts})..." + ) + time.sleep(delay) + raise last_error client.make_request = throttled_make_request - print( - "Request throttling enabled " - f"(delay {opts['min_delay']}-{opts['max_delay']}s, " - f"pause every {opts['burst_pause_every']} requests)" - ) + if throttle_opts: + print( + "Request throttling enabled " + f"(delay {throttle_opts['min_delay']}-{throttle_opts['max_delay']}s, " + f"pause every {throttle_opts['burst_pause_every']} requests)" + ) + if retry_opts: + print( + "Request retries enabled " + f"(up to {retry_opts['max_attempts']} attempts, " + f"backoff from {retry_opts['base_delay']}s)" + ) def connect(host: str, login_token: str = None, username: str = None, password: str = None) -> Driver: @@ -364,6 +419,15 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out # Create output directory output_base = pathlib.Path(output_base) / channel_name output_base.mkdir(parents=True, exist_ok=True) + + existing_post_ids = set() + if merge and output_filepath.exists(): + with open(output_filepath, encoding='utf8') as f: + existing_export = json.load(f) + existing_post_ids = {post["id"] for post in existing_export.get("posts", [])} + if existing_post_ids: + print(f" skipping {len(existing_post_ids)} posts already exported") + # Simplify all posts to contain only username, date, message and files in chronological order simple_posts = [] for i_post, post in enumerate(reversed(all_posts)): @@ -373,6 +437,9 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out if (before_ts and created > before_ts) or (after_ts and created <= after_ts): continue + if post["id"] in existing_post_ids: + continue + user_id = post["user_id"] if user_id not in user_id_to_name: user_id_to_name[user_id] = d.users.get_user(user_id)["username"] @@ -402,20 +469,24 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out for file in post["metadata"]["files"]: if download_files: filename = "%03d" % i_post + "_" + file["name"] - print("Downloading", file["name"]) - try: - resp = d.files.get_file(file["id"]) - except ResourceNotFound: - print(f"File not found, skipping: {file['name']}") - filenames.append(file["name"]) - continue - # Mattermost Driver unfortunately parses json files to dicts - if isinstance(resp, dict): - with open(output_base / filename, "w") as f: - json.dump(resp, f) + dest = output_base / filename + if merge and dest.exists(): + print(f"Skipping already downloaded {file['name']}") else: - with open(output_base / filename, "wb") as f: - f.write(resp.content) + print("Downloading", file["name"]) + try: + resp = d.files.get_file(file["id"]) + except ResourceNotFound: + print(f"File not found, skipping: {file['name']}") + filenames.append(file["name"]) + continue + # Mattermost Driver unfortunately parses json files to dicts + if isinstance(resp, dict): + with open(dest, "w") as f: + json.dump(resp, f) + else: + with open(dest, "wb") as f: + f.write(resp.content) filenames.append(file["name"]) simple_post["files"] = filenames