From 0b87a1b339e71d8d6dcecf1da0c80316c8d223f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9C=D0=B0=D1=80?= =?UTF-8?q?=D0=B8=D0=BD=D0=BA=D0=B5=D0=B2=D0=B8=D1=87?= Date: Tue, 4 Aug 2026 15:24:09 +0700 Subject: [PATCH] Skip idle channels in incremental export via last_post_at. Avoids a posts?since= round-trip for channels with no new activity since the last dump. Co-authored-by: Cursor --- mattermost-dl.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/mattermost-dl.py b/mattermost-dl.py index 083ea9b..63d3348 100644 --- a/mattermost-dl.py +++ b/mattermost-dl.py @@ -386,6 +386,45 @@ def find_channels(channels: List[dict], channel_spec: str) -> List[dict]: return selected +def _channel_export_filepath(output_base: str, channel: dict) -> pathlib.Path: + channel_name = channel["display_name"].replace("\\", "").replace("/", "") + filtered = "".join(ch for ch in channel_name if ch not in "?!/\\.;:*\"<>|") + return pathlib.Path(output_base) / channel_name / (filtered + ".json") + + +def filter_channels_by_last_post( + channels: List[dict], + after: str, + output_base: str, + merge: bool = False, +) -> List[dict]: + """Drop idle channels using Mattermost `last_post_at` before posts fetch. + + Skips channels whose latest post is not newer than `after`, avoiding a + per-channel `posts?since=` round-trip. In merge mode, channels without a + local export are always kept (first-time full history). Note: `last_post_at` + does not advance on edits to older posts — those still need `posts?since=`. + """ + after_ms = int(_parse_filter_timestamp(after) * 1000) + selected = [] + skipped = 0 + for channel in channels: + if merge and not _channel_export_filepath(output_base, channel).exists(): + selected.append(channel) + continue + last_post_at = channel.get("last_post_at") or 0 + if last_post_at > after_ms: + selected.append(channel) + else: + skipped += 1 + if skipped: + print( + f"Skipping {skipped} channel(s) with last_post_at <= {after} " + f"({len(selected)} remaining)" + ) + return selected + + def select_team(d: Driver, my_user_id: str) -> str: print("Downloading all team information... ", end="") teams = d.teams.get_user_teams(my_user_id) @@ -460,7 +499,7 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out after_ts = _parse_filter_timestamp(after) if after else None before_ts = _parse_filter_timestamp(before) if before else None filtered_channel_name = ''.join(filter(lambda ch: ch not in "?!/\\.;:*\"<>|", channel_name)) - output_filepath = pathlib.Path(output_base) / channel_name / (filtered_channel_name + ".json") + output_filepath = _channel_export_filepath(output_base, channel) if merge and after_ts is not None and not output_filepath.exists(): print(" no existing export for this channel, fetching full history") after_ts = None @@ -898,6 +937,8 @@ if __name__ == '__main__': else: team = select_team(d, my_user_id) channels = select_channel(d, team, my_user_id, user_id_to_name) + if after: + channels = filter_channels_by_last_post(channels, after, output_base, merge=merge) for i_channel, channel in enumerate(channels): print(f"Start export of channel {i_channel + 1}/{len(channels)}") export_channel(d, channel, user_id_to_name, output_base, config["download_files"],