Skip idle channels in incremental export via last_post_at.
Avoids a posts?since= round-trip for channels with no new activity since the last dump. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+42
-1
@@ -386,6 +386,45 @@ def find_channels(channels: List[dict], channel_spec: str) -> List[dict]:
|
||||
return selected
|
||||
|
||||
|
||||
def _channel_export_filepath(output_base: str, channel: dict) -> pathlib.Path:
|
||||
channel_name = channel["display_name"].replace("\\", "").replace("/", "")
|
||||
filtered = "".join(ch for ch in channel_name if ch not in "?!/\\.;:*\"<>|")
|
||||
return pathlib.Path(output_base) / channel_name / (filtered + ".json")
|
||||
|
||||
|
||||
def filter_channels_by_last_post(
|
||||
channels: List[dict],
|
||||
after: str,
|
||||
output_base: str,
|
||||
merge: bool = False,
|
||||
) -> List[dict]:
|
||||
"""Drop idle channels using Mattermost `last_post_at` before posts fetch.
|
||||
|
||||
Skips channels whose latest post is not newer than `after`, avoiding a
|
||||
per-channel `posts?since=` round-trip. In merge mode, channels without a
|
||||
local export are always kept (first-time full history). Note: `last_post_at`
|
||||
does not advance on edits to older posts — those still need `posts?since=`.
|
||||
"""
|
||||
after_ms = int(_parse_filter_timestamp(after) * 1000)
|
||||
selected = []
|
||||
skipped = 0
|
||||
for channel in channels:
|
||||
if merge and not _channel_export_filepath(output_base, channel).exists():
|
||||
selected.append(channel)
|
||||
continue
|
||||
last_post_at = channel.get("last_post_at") or 0
|
||||
if last_post_at > after_ms:
|
||||
selected.append(channel)
|
||||
else:
|
||||
skipped += 1
|
||||
if skipped:
|
||||
print(
|
||||
f"Skipping {skipped} channel(s) with last_post_at <= {after} "
|
||||
f"({len(selected)} remaining)"
|
||||
)
|
||||
return selected
|
||||
|
||||
|
||||
def select_team(d: Driver, my_user_id: str) -> str:
|
||||
print("Downloading all team information... ", end="")
|
||||
teams = d.teams.get_user_teams(my_user_id)
|
||||
@@ -460,7 +499,7 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out
|
||||
after_ts = _parse_filter_timestamp(after) if after else None
|
||||
before_ts = _parse_filter_timestamp(before) if before else None
|
||||
filtered_channel_name = ''.join(filter(lambda ch: ch not in "?!/\\.;:*\"<>|", channel_name))
|
||||
output_filepath = pathlib.Path(output_base) / channel_name / (filtered_channel_name + ".json")
|
||||
output_filepath = _channel_export_filepath(output_base, channel)
|
||||
if merge and after_ts is not None and not output_filepath.exists():
|
||||
print(" no existing export for this channel, fetching full history")
|
||||
after_ts = None
|
||||
@@ -898,6 +937,8 @@ if __name__ == '__main__':
|
||||
else:
|
||||
team = select_team(d, my_user_id)
|
||||
channels = select_channel(d, team, my_user_id, user_id_to_name)
|
||||
if after:
|
||||
channels = filter_channels_by_last_post(channels, after, output_base, merge=merge)
|
||||
for i_channel, channel in enumerate(channels):
|
||||
print(f"Start export of channel {i_channel + 1}/{len(channels)}")
|
||||
export_channel(d, channel, user_id_to_name, output_base, config["download_files"],
|
||||
|
||||
Reference in New Issue
Block a user