Skip idle channels in incremental export via last_post_at.

Avoids a posts?since= round-trip for channels with no new activity since the last dump.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Сергей Маринкевич
2026-08-04 15:24:09 +07:00
parent 6f8648f7fa
commit 0b87a1b339
+42 -1
View File
@@ -386,6 +386,45 @@ def find_channels(channels: List[dict], channel_spec: str) -> List[dict]:
return selected return selected
def _channel_export_filepath(output_base: str, channel: dict) -> pathlib.Path:
channel_name = channel["display_name"].replace("\\", "").replace("/", "")
filtered = "".join(ch for ch in channel_name if ch not in "?!/\\.;:*\"<>|")
return pathlib.Path(output_base) / channel_name / (filtered + ".json")
def filter_channels_by_last_post(
channels: List[dict],
after: str,
output_base: str,
merge: bool = False,
) -> List[dict]:
"""Drop idle channels using Mattermost `last_post_at` before posts fetch.
Skips channels whose latest post is not newer than `after`, avoiding a
per-channel `posts?since=` round-trip. In merge mode, channels without a
local export are always kept (first-time full history). Note: `last_post_at`
does not advance on edits to older posts — those still need `posts?since=`.
"""
after_ms = int(_parse_filter_timestamp(after) * 1000)
selected = []
skipped = 0
for channel in channels:
if merge and not _channel_export_filepath(output_base, channel).exists():
selected.append(channel)
continue
last_post_at = channel.get("last_post_at") or 0
if last_post_at > after_ms:
selected.append(channel)
else:
skipped += 1
if skipped:
print(
f"Skipping {skipped} channel(s) with last_post_at <= {after} "
f"({len(selected)} remaining)"
)
return selected
def select_team(d: Driver, my_user_id: str) -> str: def select_team(d: Driver, my_user_id: str) -> str:
print("Downloading all team information... ", end="") print("Downloading all team information... ", end="")
teams = d.teams.get_user_teams(my_user_id) teams = d.teams.get_user_teams(my_user_id)
@@ -460,7 +499,7 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out
after_ts = _parse_filter_timestamp(after) if after else None after_ts = _parse_filter_timestamp(after) if after else None
before_ts = _parse_filter_timestamp(before) if before else None before_ts = _parse_filter_timestamp(before) if before else None
filtered_channel_name = ''.join(filter(lambda ch: ch not in "?!/\\.;:*\"<>|", channel_name)) filtered_channel_name = ''.join(filter(lambda ch: ch not in "?!/\\.;:*\"<>|", channel_name))
output_filepath = pathlib.Path(output_base) / channel_name / (filtered_channel_name + ".json") output_filepath = _channel_export_filepath(output_base, channel)
if merge and after_ts is not None and not output_filepath.exists(): if merge and after_ts is not None and not output_filepath.exists():
print(" no existing export for this channel, fetching full history") print(" no existing export for this channel, fetching full history")
after_ts = None after_ts = None
@@ -898,6 +937,8 @@ if __name__ == '__main__':
else: else:
team = select_team(d, my_user_id) team = select_team(d, my_user_id)
channels = select_channel(d, team, my_user_id, user_id_to_name) channels = select_channel(d, team, my_user_id, user_id_to_name)
if after:
channels = filter_channels_by_last_post(channels, after, output_base, merge=merge)
for i_channel, channel in enumerate(channels): for i_channel, channel in enumerate(channels):
print(f"Start export of channel {i_channel + 1}/{len(channels)}") print(f"Start export of channel {i_channel + 1}/{len(channels)}")
export_channel(d, channel, user_id_to_name, output_base, config["download_files"], export_channel(d, channel, user_id_to_name, output_base, config["download_files"],