Skip idle channels in incremental export via last_post_at.
Avoids a posts?since= round-trip for channels with no new activity since the last dump. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+42
-1
@@ -386,6 +386,45 @@ def find_channels(channels: List[dict], channel_spec: str) -> List[dict]:
|
|||||||
return selected
|
return selected
|
||||||
|
|
||||||
|
|
||||||
|
def _channel_export_filepath(output_base: str, channel: dict) -> pathlib.Path:
|
||||||
|
channel_name = channel["display_name"].replace("\\", "").replace("/", "")
|
||||||
|
filtered = "".join(ch for ch in channel_name if ch not in "?!/\\.;:*\"<>|")
|
||||||
|
return pathlib.Path(output_base) / channel_name / (filtered + ".json")
|
||||||
|
|
||||||
|
|
||||||
|
def filter_channels_by_last_post(
|
||||||
|
channels: List[dict],
|
||||||
|
after: str,
|
||||||
|
output_base: str,
|
||||||
|
merge: bool = False,
|
||||||
|
) -> List[dict]:
|
||||||
|
"""Drop idle channels using Mattermost `last_post_at` before posts fetch.
|
||||||
|
|
||||||
|
Skips channels whose latest post is not newer than `after`, avoiding a
|
||||||
|
per-channel `posts?since=` round-trip. In merge mode, channels without a
|
||||||
|
local export are always kept (first-time full history). Note: `last_post_at`
|
||||||
|
does not advance on edits to older posts — those still need `posts?since=`.
|
||||||
|
"""
|
||||||
|
after_ms = int(_parse_filter_timestamp(after) * 1000)
|
||||||
|
selected = []
|
||||||
|
skipped = 0
|
||||||
|
for channel in channels:
|
||||||
|
if merge and not _channel_export_filepath(output_base, channel).exists():
|
||||||
|
selected.append(channel)
|
||||||
|
continue
|
||||||
|
last_post_at = channel.get("last_post_at") or 0
|
||||||
|
if last_post_at > after_ms:
|
||||||
|
selected.append(channel)
|
||||||
|
else:
|
||||||
|
skipped += 1
|
||||||
|
if skipped:
|
||||||
|
print(
|
||||||
|
f"Skipping {skipped} channel(s) with last_post_at <= {after} "
|
||||||
|
f"({len(selected)} remaining)"
|
||||||
|
)
|
||||||
|
return selected
|
||||||
|
|
||||||
|
|
||||||
def select_team(d: Driver, my_user_id: str) -> str:
|
def select_team(d: Driver, my_user_id: str) -> str:
|
||||||
print("Downloading all team information... ", end="")
|
print("Downloading all team information... ", end="")
|
||||||
teams = d.teams.get_user_teams(my_user_id)
|
teams = d.teams.get_user_teams(my_user_id)
|
||||||
@@ -460,7 +499,7 @@ def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], out
|
|||||||
after_ts = _parse_filter_timestamp(after) if after else None
|
after_ts = _parse_filter_timestamp(after) if after else None
|
||||||
before_ts = _parse_filter_timestamp(before) if before else None
|
before_ts = _parse_filter_timestamp(before) if before else None
|
||||||
filtered_channel_name = ''.join(filter(lambda ch: ch not in "?!/\\.;:*\"<>|", channel_name))
|
filtered_channel_name = ''.join(filter(lambda ch: ch not in "?!/\\.;:*\"<>|", channel_name))
|
||||||
output_filepath = pathlib.Path(output_base) / channel_name / (filtered_channel_name + ".json")
|
output_filepath = _channel_export_filepath(output_base, channel)
|
||||||
if merge and after_ts is not None and not output_filepath.exists():
|
if merge and after_ts is not None and not output_filepath.exists():
|
||||||
print(" no existing export for this channel, fetching full history")
|
print(" no existing export for this channel, fetching full history")
|
||||||
after_ts = None
|
after_ts = None
|
||||||
@@ -898,6 +937,8 @@ if __name__ == '__main__':
|
|||||||
else:
|
else:
|
||||||
team = select_team(d, my_user_id)
|
team = select_team(d, my_user_id)
|
||||||
channels = select_channel(d, team, my_user_id, user_id_to_name)
|
channels = select_channel(d, team, my_user_id, user_id_to_name)
|
||||||
|
if after:
|
||||||
|
channels = filter_channels_by_last_post(channels, after, output_base, merge=merge)
|
||||||
for i_channel, channel in enumerate(channels):
|
for i_channel, channel in enumerate(channels):
|
||||||
print(f"Start export of channel {i_channel + 1}/{len(channels)}")
|
print(f"Start export of channel {i_channel + 1}/{len(channels)}")
|
||||||
export_channel(d, channel, user_id_to_name, output_base, config["download_files"],
|
export_channel(d, channel, user_id_to_name, output_base, config["download_files"],
|
||||||
|
|||||||
Reference in New Issue
Block a user