Сергей Маринкевич
4820fb884b
Keep channel mtime stable when merge finds no new posts, and rebuild FTS only for new/changed content so large daily channels stay cheap. Drop --reindex-all from renew.sh for incremental runs. Co-authored-by: Cursor <cursoragent@cursor.com>
957 lines
36 KiB
Python
957 lines
36 KiB
Python
import argparse
|
|
import os
|
|
import pickle
|
|
import random
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
from datetime import datetime, date, timezone
|
|
from typing import Tuple, Dict, List, Optional
|
|
import getpass
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from mattermostdriver import Driver
|
|
from mattermostdriver.exceptions import ResourceNotFound
|
|
import pathlib
|
|
import json
|
|
|
|
GITLAB_URL_DEFAULT = "https://mmgit.eltex-co.ru"
|
|
COOKIE_FILE_DEFAULT = "cookies.pkl"
|
|
|
|
THROTTLE_DEFAULTS = {
|
|
"min_delay": 0.4,
|
|
"max_delay": 1.8,
|
|
"burst_pause_every": 12,
|
|
"burst_pause_min": 3.0,
|
|
"burst_pause_max": 8.0,
|
|
"file_delay_multiplier": 1.5,
|
|
}
|
|
|
|
RETRY_DEFAULTS = {
|
|
"max_attempts": 5,
|
|
"base_delay": 2.0,
|
|
"max_delay": 60.0,
|
|
}
|
|
|
|
RETRYABLE_REQUEST_ERRORS = (
|
|
requests.exceptions.ConnectionError,
|
|
requests.exceptions.Timeout,
|
|
requests.exceptions.ChunkedEncodingError,
|
|
)
|
|
|
|
|
|
class RequestThrottler:
|
|
"""Paces API calls with jitter and occasional longer pauses."""
|
|
|
|
def __init__(self, **opts):
|
|
self.min_delay = opts["min_delay"]
|
|
self.max_delay = opts["max_delay"]
|
|
self.burst_pause_every = opts["burst_pause_every"]
|
|
self.burst_pause_min = opts["burst_pause_min"]
|
|
self.burst_pause_max = opts["burst_pause_max"]
|
|
self.file_delay_multiplier = opts["file_delay_multiplier"]
|
|
self._request_count = 0
|
|
self._last_request_at = 0.0
|
|
|
|
def wait(self, endpoint: str = "") -> None:
|
|
self._request_count += 1
|
|
if self.burst_pause_every and self._request_count % self.burst_pause_every == 0:
|
|
delay = random.uniform(self.burst_pause_min, self.burst_pause_max)
|
|
else:
|
|
delay = random.uniform(self.min_delay, self.max_delay)
|
|
|
|
if endpoint.startswith("/files/"):
|
|
delay *= self.file_delay_multiplier
|
|
|
|
elapsed = time.monotonic() - self._last_request_at
|
|
if elapsed < delay:
|
|
time.sleep(delay - elapsed)
|
|
self._last_request_at = time.monotonic()
|
|
|
|
|
|
def _parse_retry_config(config: dict) -> Optional[dict]:
|
|
retry = config.get("retry")
|
|
if retry is False:
|
|
return None
|
|
if retry is None or retry is True:
|
|
return dict(RETRY_DEFAULTS)
|
|
if isinstance(retry, dict):
|
|
if not retry.get("enabled", True):
|
|
return None
|
|
opts = dict(RETRY_DEFAULTS)
|
|
opts.update({k: v for k, v in retry.items() if k != "enabled"})
|
|
return opts
|
|
return None
|
|
|
|
|
|
def _parse_throttle_config(config: dict) -> Optional[dict]:
|
|
throttle = config.get("throttle")
|
|
if not throttle:
|
|
return None
|
|
if throttle is True:
|
|
return dict(THROTTLE_DEFAULTS)
|
|
if isinstance(throttle, dict):
|
|
if not throttle.get("enabled", True):
|
|
return None
|
|
opts = dict(THROTTLE_DEFAULTS)
|
|
opts.update({k: v for k, v in throttle.items() if k != "enabled"})
|
|
return opts
|
|
return None
|
|
|
|
|
|
def apply_throttle(d: Driver, config: dict) -> None:
|
|
throttle_opts = _parse_throttle_config(config)
|
|
retry_opts = _parse_retry_config(config)
|
|
if not throttle_opts and not retry_opts:
|
|
return
|
|
|
|
throttler = RequestThrottler(**(throttle_opts or THROTTLE_DEFAULTS))
|
|
client = d.client
|
|
original_make_request = client.make_request
|
|
|
|
def throttled_make_request(method, endpoint, options=None, params=None, data=None, files=None, basepath=None):
|
|
if throttle_opts:
|
|
throttler.wait(endpoint)
|
|
|
|
attempts = retry_opts["max_attempts"] if retry_opts else 1
|
|
last_error = None
|
|
for attempt in range(attempts):
|
|
try:
|
|
return original_make_request(method, endpoint, options, params, data, files, basepath)
|
|
except RETRYABLE_REQUEST_ERRORS as error:
|
|
last_error = error
|
|
if attempt + 1 >= attempts:
|
|
break
|
|
delay = min(
|
|
retry_opts["base_delay"] * (2 ** attempt),
|
|
retry_opts["max_delay"],
|
|
) + random.uniform(0, 1)
|
|
print(
|
|
f"Request failed ({error}), retrying in {delay:.1f}s "
|
|
f"({attempt + 2}/{attempts})..."
|
|
)
|
|
time.sleep(delay)
|
|
raise last_error
|
|
|
|
client.make_request = throttled_make_request
|
|
if throttle_opts:
|
|
print(
|
|
"Request throttling enabled "
|
|
f"(delay {throttle_opts['min_delay']}-{throttle_opts['max_delay']}s, "
|
|
f"pause every {throttle_opts['burst_pause_every']} requests)"
|
|
)
|
|
if retry_opts:
|
|
print(
|
|
"Request retries enabled "
|
|
f"(up to {retry_opts['max_attempts']} attempts, "
|
|
f"backoff from {retry_opts['base_delay']}s)"
|
|
)
|
|
|
|
|
|
def connect(host: str, login_token: str = None, username: str = None, password: str = None) -> Driver:
|
|
d = Driver({
|
|
"url": host,
|
|
"port": 443,
|
|
"token": login_token,
|
|
"username": username,
|
|
"password": password
|
|
})
|
|
d.login()
|
|
return d
|
|
|
|
|
|
def _mattermost_url(host: str) -> str:
|
|
return f"https://{host}"
|
|
|
|
|
|
def _save_cookies(session: requests.Session, filename: str) -> None:
|
|
try:
|
|
with open(filename, "wb") as f:
|
|
pickle.dump(session.cookies, f)
|
|
except OSError as e:
|
|
print(f"Warning: failed to save cookies to {filename}: {e}")
|
|
|
|
|
|
def _load_cookies(session: requests.Session, filename: str) -> bool:
|
|
if not os.path.exists(filename):
|
|
return False
|
|
try:
|
|
with open(filename, "rb") as f:
|
|
session.cookies.update(pickle.load(f))
|
|
return True
|
|
except (OSError, pickle.PickleError) as e:
|
|
print(f"Warning: failed to load cookies from {filename}: {e}")
|
|
return False
|
|
|
|
|
|
def _remove_cookie_file(filename: str) -> None:
|
|
try:
|
|
if os.path.exists(filename):
|
|
os.remove(filename)
|
|
print(f"Removed invalid cookie file: {filename}")
|
|
except OSError as e:
|
|
print(f"Warning: failed to remove cookie file {filename}: {e}")
|
|
|
|
|
|
def _get_current_user(session: requests.Session, host: str) -> Optional[dict]:
|
|
try:
|
|
response = session.get(f"{_mattermost_url(host)}/api/v4/users/me", timeout=10)
|
|
return response.json() if response.status_code == 200 else None
|
|
except requests.exceptions.RequestException:
|
|
return None
|
|
|
|
|
|
def _login_to_mattermost_via_gitlab(session: requests.Session, host: str, gitlab_url: str,
|
|
username: str, password: str) -> bool:
|
|
login_url = f"{_mattermost_url(host)}/oauth/gitlab/login"
|
|
headers = {"User-Agent": "Mozilla/5.0"}
|
|
|
|
try:
|
|
response = session.get(login_url, headers=headers, allow_redirects=True, timeout=15)
|
|
response.raise_for_status()
|
|
|
|
if "/users/sign_in" in response.url or "/users/auth/ldapmain" in response.url:
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
auth_form = soup.find("form", {"id": "new_ldap_user"})
|
|
if not auth_form:
|
|
return False
|
|
auth_token_input = auth_form.find("input", {"name": "authenticity_token"})
|
|
if not auth_token_input:
|
|
return False
|
|
|
|
auth_token = auth_token_input["value"]
|
|
payload = {"authenticity_token": auth_token, "username": username, "password": password}
|
|
callback_url = f"{gitlab_url}/users/auth/ldapmain/callback"
|
|
response = session.post(
|
|
callback_url, data=payload, headers={"Referer": response.url},
|
|
allow_redirects=True, timeout=15
|
|
)
|
|
response.raise_for_status()
|
|
|
|
if response.status_code == 200 and "/oauth/authorize" in response.url:
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
redirect_link = soup.find("a", href=True)
|
|
if not redirect_link:
|
|
return False
|
|
|
|
response = session.get(
|
|
redirect_link["href"], headers={"Referer": response.url},
|
|
allow_redirects=True, timeout=15
|
|
)
|
|
response.raise_for_status()
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f"GitLab login failed: {e}")
|
|
return False
|
|
|
|
return "MMAUTHTOKEN" in session.cookies
|
|
|
|
|
|
def try_get_token_from_cookies(host: str, cookie_file: str = COOKIE_FILE_DEFAULT) -> Optional[str]:
|
|
session = requests.Session()
|
|
if not _load_cookies(session, cookie_file):
|
|
return None
|
|
if not _get_current_user(session, host):
|
|
return None
|
|
return session.cookies.get("MMAUTHTOKEN")
|
|
|
|
|
|
def get_token_via_gitlab(host: str, gitlab_url: str, username: str, password: str,
|
|
cookie_file: str = COOKIE_FILE_DEFAULT) -> Optional[str]:
|
|
session = requests.Session()
|
|
|
|
if _load_cookies(session, cookie_file):
|
|
print("Checking saved GitLab/Mattermost session...")
|
|
if _get_current_user(session, host):
|
|
token = session.cookies.get("MMAUTHTOKEN")
|
|
if token:
|
|
print("Authenticated using saved cookies.")
|
|
return token
|
|
print("Saved cookies are invalid or expired.")
|
|
_remove_cookie_file(cookie_file)
|
|
|
|
print("Logging in to Mattermost via GitLab...")
|
|
if not _login_to_mattermost_via_gitlab(session, host, gitlab_url, username, password):
|
|
print("Failed to authenticate via GitLab.")
|
|
return None
|
|
|
|
_save_cookies(session, cookie_file)
|
|
print(f"Session saved to {cookie_file}")
|
|
return session.cookies.get("MMAUTHTOKEN")
|
|
|
|
|
|
def get_users(d: Driver) -> Tuple[Dict[str, str], str, List[dict]]:
|
|
my_user = d.users.get_user("me")
|
|
my_username = my_user["username"]
|
|
my_user_id = my_user["id"]
|
|
print(f"Successfully logged in as {my_username} ({my_user_id})")
|
|
|
|
# Get all usernames as we want to use those instead of the user ids
|
|
user_id_to_name = {}
|
|
user_profiles = []
|
|
page = 0
|
|
print("Downloading all user information... ", end="")
|
|
while True:
|
|
users_resp = d.users.get_users(params={"per_page": 200, "page": page})
|
|
if len(users_resp) == 0:
|
|
break
|
|
for user in users_resp:
|
|
user_id_to_name[user["id"]] = user["username"]
|
|
user_profiles.append(
|
|
{
|
|
"username": user.get("username") or "",
|
|
"first_name": user.get("first_name") or "",
|
|
"last_name": user.get("last_name") or "",
|
|
"nickname": user.get("nickname") or "",
|
|
}
|
|
)
|
|
page += 1
|
|
print(f"Found {len(user_id_to_name)} users!")
|
|
|
|
return user_id_to_name, my_user_id, user_profiles
|
|
|
|
|
|
def save_users_json(output_base: str, user_profiles: List[dict]) -> str:
|
|
output_path = pathlib.Path(output_base) / "users.json"
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(output_path, "w", encoding="utf-8") as f:
|
|
json.dump(user_profiles, f, indent=2, ensure_ascii=False)
|
|
print(f"Saved user directory to '{output_path}'")
|
|
return str(output_path)
|
|
|
|
|
|
def _enrich_direct_message_channels(d: Driver, channels: List[dict], my_user_id: str,
|
|
user_id_to_name: Dict[str, str]) -> None:
|
|
for channel in channels:
|
|
if channel["type"] != "D":
|
|
continue
|
|
|
|
user_ids = channel["name"].split("__")
|
|
other_user_id = user_ids[1] if user_ids[0] == my_user_id else user_ids[0]
|
|
if other_user_id not in user_id_to_name:
|
|
try:
|
|
user_id_to_name[other_user_id] = d.users.get_user(other_user_id)["username"]
|
|
except Exception:
|
|
user_id_to_name[other_user_id] = channel.get("display_name") or other_user_id
|
|
channel["display_name"] = user_id_to_name[other_user_id]
|
|
|
|
|
|
def get_channels_for_team(d: Driver, team: dict, my_user_id: str,
|
|
user_id_to_name: Dict[str, str]) -> List[dict]:
|
|
channels = d.channels.get_channels_for_user(my_user_id, team["id"])
|
|
_enrich_direct_message_channels(d, channels, my_user_id, user_id_to_name)
|
|
return sorted(channels, key=lambda x: x["display_name"].lower())
|
|
|
|
|
|
def find_team(d: Driver, my_user_id: str, team_name: str) -> dict:
|
|
teams = d.teams.get_user_teams(my_user_id)
|
|
for team in teams:
|
|
if team["name"] == team_name:
|
|
return team
|
|
available = ", ".join(team["name"] for team in teams)
|
|
raise SystemExit(f"Team {team_name!r} not found. Available teams: {available}")
|
|
|
|
|
|
def find_channels(channels: List[dict], channel_spec: str) -> List[dict]:
|
|
if channel_spec.lower() == "all":
|
|
return channels
|
|
|
|
requested = [part.strip() for part in channel_spec.split(",") if part.strip()]
|
|
if not requested:
|
|
raise SystemExit("No channels specified (use channel names or 'all').")
|
|
|
|
by_name = {}
|
|
by_display = {}
|
|
for channel in channels:
|
|
by_name[channel["name"].lower()] = channel
|
|
by_display[channel["display_name"].lower()] = channel
|
|
|
|
selected = []
|
|
missing = []
|
|
for name in requested:
|
|
key = name.lower()
|
|
channel = by_name.get(key) or by_display.get(key)
|
|
if channel is None:
|
|
missing.append(name)
|
|
else:
|
|
selected.append(channel)
|
|
|
|
if missing:
|
|
available = ", ".join(channel["display_name"] for channel in channels)
|
|
raise SystemExit(
|
|
f"Channel(s) not found: {', '.join(missing)}. "
|
|
f"Available channels: {available}"
|
|
)
|
|
return selected
|
|
|
|
|
|
def _channel_export_filepath(output_base: str, channel: dict) -> pathlib.Path:
|
|
channel_name = channel["display_name"].replace("\\", "").replace("/", "")
|
|
filtered = "".join(ch for ch in channel_name if ch not in "?!/\\.;:*\"<>|")
|
|
return pathlib.Path(output_base) / channel_name / (filtered + ".json")
|
|
|
|
|
|
def filter_channels_by_last_post(
|
|
channels: List[dict],
|
|
after: str,
|
|
output_base: str,
|
|
merge: bool = False,
|
|
) -> List[dict]:
|
|
"""Drop idle channels using Mattermost `last_post_at` before posts fetch.
|
|
|
|
Skips channels whose latest post is not newer than `after`, avoiding a
|
|
per-channel `posts?since=` round-trip. In merge mode, channels without a
|
|
local export are always kept (first-time full history). Note: `last_post_at`
|
|
does not advance on edits to older posts — those still need `posts?since=`.
|
|
"""
|
|
after_ms = int(_parse_filter_timestamp(after) * 1000)
|
|
selected = []
|
|
skipped = 0
|
|
for channel in channels:
|
|
if merge and not _channel_export_filepath(output_base, channel).exists():
|
|
selected.append(channel)
|
|
continue
|
|
last_post_at = channel.get("last_post_at") or 0
|
|
if last_post_at > after_ms:
|
|
selected.append(channel)
|
|
else:
|
|
skipped += 1
|
|
if skipped:
|
|
print(
|
|
f"Skipping {skipped} channel(s) with last_post_at <= {after} "
|
|
f"({len(selected)} remaining)"
|
|
)
|
|
return selected
|
|
|
|
|
|
def select_team(d: Driver, my_user_id: str) -> str:
|
|
print("Downloading all team information... ", end="")
|
|
teams = d.teams.get_user_teams(my_user_id)
|
|
print(f"Found {len(teams)} teams!")
|
|
for i_team, team in enumerate(teams):
|
|
print(f"{i_team}\t{team['name']}\t({team['id']})")
|
|
team_idx = int(input("Select team by idx: "))
|
|
team = teams[team_idx]
|
|
print(f"Selected team {team['name']}")
|
|
return team
|
|
|
|
|
|
def select_channel(d: Driver, team: str, my_user_id: str, user_id_to_name: Dict[str, str],
|
|
verbose: bool = False) -> List[str]:
|
|
print("Downloading all channel information... ", end="")
|
|
channels = get_channels_for_team(d, team, my_user_id, user_id_to_name)
|
|
print(f"Found {len(channels)} channels!")
|
|
|
|
for i_channel, channel in enumerate(channels):
|
|
if verbose:
|
|
channel_id = f"\t({channel['id']})"
|
|
else:
|
|
channel_id = ""
|
|
print(f"{i_channel}\t{channel['display_name']}{channel_id}")
|
|
channel_input = input("Select channels by idx separated by comma or type 'all' for downloading all channels: ")
|
|
if channel_input == "all":
|
|
channel_idxs = list(range(len(channels)))
|
|
else:
|
|
channel_idxs = channel_input.replace(" ", "").split(",")
|
|
selected_channels = [channels[int(idx)] for idx in channel_idxs]
|
|
print("Selected channel(s):", ", ".join([channel["display_name"] for channel in selected_channels]))
|
|
return selected_channels
|
|
|
|
|
|
def _resolve_team_name(d: Driver, channel: dict, fallback_team: Optional[dict] = None) -> str:
|
|
team_id = channel.get("team_id") or ""
|
|
if team_id:
|
|
try:
|
|
return d.teams.get_team(team_id)["name"]
|
|
except ResourceNotFound:
|
|
pass
|
|
if fallback_team:
|
|
return fallback_team["name"]
|
|
return ""
|
|
|
|
|
|
def _parse_filter_timestamp(s: str) -> float:
|
|
for fmt in ('%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%d'):
|
|
try:
|
|
dt = datetime.strptime(s, fmt)
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
return dt.timestamp()
|
|
except ValueError:
|
|
continue
|
|
raise ValueError(f"Invalid date format: {s!r} (expected YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ)")
|
|
|
|
|
|
def save_config(config: dict, config_filename: str = "config.json") -> None:
|
|
to_save = {k: v for k, v in config.items() if k != "password"}
|
|
with open(config_filename, "w") as f:
|
|
json.dump(to_save, f, indent=2)
|
|
|
|
|
|
def export_channel(d: Driver, channel: str, user_id_to_name: Dict[str, str], output_base: str,
|
|
download_files: bool = True, before: str = None, after: str = None,
|
|
team: Optional[dict] = None, merge: bool = False):
|
|
# Sanitize channel name
|
|
channel_name = channel["display_name"].replace("\\", "").replace("/", "")
|
|
|
|
print("Exporting channel", channel_name)
|
|
after_ts = _parse_filter_timestamp(after) if after else None
|
|
before_ts = _parse_filter_timestamp(before) if before else None
|
|
filtered_channel_name = ''.join(filter(lambda ch: ch not in "?!/\\.;:*\"<>|", channel_name))
|
|
output_filepath = _channel_export_filepath(output_base, channel)
|
|
if merge and after_ts is not None and not output_filepath.exists():
|
|
print(" no existing export for this channel, fetching full history")
|
|
after_ts = None
|
|
elif after:
|
|
print(f" fetching posts after {after}")
|
|
|
|
# Get all posts for selected channel
|
|
page = 0
|
|
all_posts = []
|
|
while True:
|
|
print(f"Requesting channel page {page}")
|
|
params = {"per_page": 200, "page": page}
|
|
if after_ts is not None:
|
|
# Ask server only for posts changed since last export.
|
|
# For incremental updates we typically expect few new posts,
|
|
# so a single page is enough and avoids walking the whole history.
|
|
params["since"] = int(after_ts * 1000)
|
|
posts = d.posts.get_posts_for_channel(channel["id"], params=params)
|
|
|
|
if len(posts["posts"]) == 0:
|
|
# If no posts are returned, we have reached the end
|
|
break
|
|
|
|
all_posts.extend([posts["posts"][post] for post in posts["order"]])
|
|
|
|
# In incremental mode (after_ts is set) we intentionally stop
|
|
# after the first non-empty page to avoid downloading the
|
|
# entire channel history again. For full dumps, keep paging.
|
|
if after_ts is not None:
|
|
break
|
|
|
|
page += 1
|
|
print(f"Found {len(all_posts)} posts")
|
|
# Create output directory
|
|
output_base = pathlib.Path(output_base) / channel_name
|
|
output_base.mkdir(parents=True, exist_ok=True)
|
|
|
|
existing_post_ids = set()
|
|
if merge and output_filepath.exists():
|
|
with open(output_filepath, encoding='utf8') as f:
|
|
existing_export = json.load(f)
|
|
existing_post_ids = {post["id"] for post in existing_export.get("posts", [])}
|
|
if existing_post_ids:
|
|
print(f" skipping {len(existing_post_ids)} posts already exported")
|
|
|
|
# Simplify all posts to contain only username, date, message and files in chronological order
|
|
simple_posts = []
|
|
for i_post, post in enumerate(reversed(all_posts)):
|
|
|
|
# Filter posts by date range
|
|
created = post["create_at"] / 1000
|
|
if (before_ts and created > before_ts) or (after_ts and created <= after_ts):
|
|
continue
|
|
|
|
if post["id"] in existing_post_ids:
|
|
continue
|
|
|
|
user_id = post["user_id"]
|
|
if user_id not in user_id_to_name:
|
|
user_id_to_name[user_id] = d.users.get_user(user_id)["username"]
|
|
username = user_id_to_name[user_id]
|
|
created = datetime.fromtimestamp(
|
|
post["create_at"] / 1000, timezone.utc
|
|
).strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
message = post["message"]
|
|
simple_post = dict(idx=i_post, id=post["id"], created=created, username=username, message=message)
|
|
|
|
# If a code block is given in the message, dump it to file
|
|
if message.count("```") > 1:
|
|
start_pos = message.find("```") + 3
|
|
end_pos = message.rfind("```")
|
|
|
|
cut = message[start_pos:end_pos]
|
|
if not len(cut):
|
|
print("Code has no length")
|
|
else:
|
|
filename = "%03d" % i_post + "_code.txt"
|
|
with open(output_base / filename, "wb") as f:
|
|
f.write(cut.encode())
|
|
|
|
# If any files are attached to the message, download each
|
|
if "files" in post["metadata"]:
|
|
filenames = []
|
|
for file in post["metadata"]["files"]:
|
|
if download_files:
|
|
filename = "%03d" % i_post + "_" + file["name"]
|
|
dest = output_base / filename
|
|
if merge and dest.exists():
|
|
print(f"Skipping already downloaded {file['name']}")
|
|
else:
|
|
print("Downloading", file["name"])
|
|
try:
|
|
resp = d.files.get_file(file["id"])
|
|
except ResourceNotFound:
|
|
print(f"File not found, skipping: {file['name']}")
|
|
filenames.append(file["name"])
|
|
continue
|
|
# Mattermost Driver unfortunately parses json files to dicts
|
|
if isinstance(resp, dict):
|
|
with open(dest, "w") as f:
|
|
json.dump(resp, f)
|
|
else:
|
|
with open(dest, "wb") as f:
|
|
f.write(resp.content)
|
|
|
|
filenames.append(file["name"])
|
|
simple_post["files"] = filenames
|
|
simple_posts.append(simple_post)
|
|
|
|
output_filename = filtered_channel_name + ".json"
|
|
output_filepath = output_base / output_filename
|
|
|
|
merged_posts = simple_posts
|
|
if merge and output_filepath.exists():
|
|
with open(output_filepath, encoding='utf8') as f:
|
|
existing = json.load(f)
|
|
seen_ids = {post["id"] for post in existing.get("posts", [])}
|
|
merged_posts = list(existing.get("posts", []))
|
|
new_count = 0
|
|
for post in simple_posts:
|
|
if post["id"] not in seen_ids:
|
|
merged_posts.append(post)
|
|
seen_ids.add(post["id"])
|
|
new_count += 1
|
|
if new_count == 0:
|
|
# Keep mtime stable so mm_index can skip this channel file.
|
|
print(f" no new posts, leaving '{output_filepath}' unchanged")
|
|
return
|
|
merged_posts.sort(key=lambda p: p["created"])
|
|
for i, post in enumerate(merged_posts):
|
|
post["idx"] = i
|
|
print(f" merged {new_count} new posts ({len(merged_posts)} total)")
|
|
elif merge:
|
|
print(f" no existing export at '{output_filepath}', writing fresh")
|
|
|
|
output = {
|
|
"channel": {
|
|
"name": channel["name"],
|
|
"display_name": channel["display_name"],
|
|
"header": channel["header"],
|
|
"id": channel["id"],
|
|
"team": _resolve_team_name(d, channel, team),
|
|
"team_id": channel.get("team_id") or "",
|
|
"exported_at": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
},
|
|
"posts": merged_posts
|
|
}
|
|
|
|
with open(output_filepath, "w", encoding='utf8') as f:
|
|
json.dump(output, f, indent=2, ensure_ascii=False)
|
|
print(f"Exported channel data to '{output_filepath}'")
|
|
|
|
|
|
def get_config_from_json(config_filename: str = "config.json") -> dict:
|
|
config_path = pathlib.Path(config_filename)
|
|
if not config_path.exists():
|
|
return {}
|
|
|
|
with config_path.open() as f:
|
|
config = json.load(f)
|
|
|
|
return config
|
|
|
|
|
|
def complete_config_noninteractive(config: dict) -> dict:
|
|
required = ("host", "login_mode")
|
|
missing = [key for key in required if not config.get(key)]
|
|
if missing:
|
|
raise SystemExit(
|
|
f"Missing required config field(s) for non-interactive mode: {', '.join(missing)}"
|
|
)
|
|
|
|
login_mode = config["login_mode"]
|
|
if login_mode not in ("password", "token", "gitlab"):
|
|
raise SystemExit(f"Unsupported login_mode in config: {login_mode!r}")
|
|
|
|
token = config.get("token")
|
|
if login_mode == "gitlab":
|
|
token = token or try_get_token_from_cookies(
|
|
config["host"], config.get("cookie_file", COOKIE_FILE_DEFAULT)
|
|
)
|
|
if token:
|
|
config["token"] = token
|
|
elif login_mode == "password":
|
|
if not config.get("username") or not config.get("password"):
|
|
raise SystemExit(
|
|
"Non-interactive password login requires 'username' and 'password' in config."
|
|
)
|
|
else:
|
|
raise SystemExit(
|
|
"No valid token in config or cookies. "
|
|
"Run interactively once to authenticate."
|
|
)
|
|
|
|
if "download_files" not in config:
|
|
config["download_files"] = True
|
|
|
|
config["password"] = None
|
|
return config
|
|
|
|
|
|
def complete_config(config: dict, config_filename: str = "config.json", *, interactive: bool = True) -> dict:
|
|
if not interactive:
|
|
return complete_config_noninteractive(config)
|
|
|
|
config_changed = False
|
|
if config.get("host", False):
|
|
print(f"Using host '{config['host']}' from config")
|
|
else:
|
|
config["host"] = input("Please input host/server address (without https://): ")
|
|
config_changed = True
|
|
|
|
if config.get("login_mode", False):
|
|
print(f"Using login mode '{config['login_mode']}' from config")
|
|
else:
|
|
login_mode = ""
|
|
while login_mode not in ["password", "token", "gitlab"]:
|
|
login_mode = input(
|
|
"Please input login_mode 'password', 'token', or 'gitlab' (GitLab OAuth): "
|
|
)
|
|
config["login_mode"] = login_mode
|
|
config_changed = True
|
|
|
|
password = None
|
|
if config["login_mode"] == "password":
|
|
if config.get("username", False):
|
|
print(f"Using username '{config['username']}' from config")
|
|
else:
|
|
config["username"] = input("Please input your username: ")
|
|
config_changed = True
|
|
|
|
password = getpass.getpass("Enter your password (hidden): ")
|
|
elif config["login_mode"] == "gitlab":
|
|
# If we already have a token in config, reuse it and do not ask
|
|
# for GitLab credentials again.
|
|
if config.get("token"):
|
|
print("Using existing token from config for GitLab login_mode.")
|
|
else:
|
|
if config.get("gitlab_url", False):
|
|
print(f"Using GitLab URL '{config['gitlab_url']}' from config")
|
|
else:
|
|
config["gitlab_url"] = input(
|
|
f"Please input GitLab URL [{GITLAB_URL_DEFAULT}]: "
|
|
) or GITLAB_URL_DEFAULT
|
|
config_changed = True
|
|
|
|
if config.get("username", False):
|
|
print(f"Using username '{config['username']}' from config")
|
|
else:
|
|
config["username"] = input("Please input your GitLab username: ")
|
|
config_changed = True
|
|
|
|
password = getpass.getpass("Enter your GitLab password (hidden): ")
|
|
token = get_token_via_gitlab(
|
|
config["host"], config["gitlab_url"], config["username"], password,
|
|
config.get("cookie_file", COOKIE_FILE_DEFAULT)
|
|
)
|
|
if not token:
|
|
raise SystemExit("GitLab authentication failed.")
|
|
config["token"] = token
|
|
config_changed = True
|
|
else:
|
|
if config.get("token", False):
|
|
print(f"Using token '{config['token']}' from config")
|
|
else:
|
|
print("Token not found in config. Choose how to obtain MMAUTHTOKEN:")
|
|
print(" g - login via GitLab (username/password)")
|
|
print(" f - extract from Firefox cookies")
|
|
print(" m - enter token manually")
|
|
dec = ""
|
|
while dec not in ["g", "f", "m"]:
|
|
dec = input("Choice [g/f/m]: ")
|
|
|
|
token = None
|
|
if dec == "g":
|
|
if config.get("gitlab_url", False):
|
|
print(f"Using GitLab URL '{config['gitlab_url']}' from config")
|
|
else:
|
|
config["gitlab_url"] = input(
|
|
f"Please input GitLab URL [{GITLAB_URL_DEFAULT}]: "
|
|
) or GITLAB_URL_DEFAULT
|
|
config_changed = True
|
|
if config.get("username", False):
|
|
print(f"Using username '{config['username']}' from config")
|
|
else:
|
|
config["username"] = input("Please input your GitLab username: ")
|
|
config_changed = True
|
|
password = getpass.getpass("Enter your GitLab password (hidden): ")
|
|
token = get_token_via_gitlab(
|
|
config["host"], config["gitlab_url"], config["username"], password,
|
|
config.get("cookie_file", COOKIE_FILE_DEFAULT)
|
|
)
|
|
if not token:
|
|
raise SystemExit("GitLab authentication failed.")
|
|
elif dec == "f":
|
|
token = find_mmauthtoken_firefox(config["host"])
|
|
if not token:
|
|
token = input("Please input your login token (MMAUTHTOKEN): ")
|
|
config["token"] = token
|
|
config_changed = True
|
|
|
|
if "download_files" in config:
|
|
print(f"Download files set to '{config['download_files']}' from config")
|
|
else:
|
|
dec = ""
|
|
while not (dec == "y" or dec == "n"):
|
|
dec = input("Should files be downloaded? y/n: ")
|
|
config["download_files"] = dec == "y"
|
|
config_changed = True
|
|
|
|
if config_changed:
|
|
dec = ""
|
|
while not (dec == "y" or dec == "n"):
|
|
dec = input("Config changed! Would you like to store your config (without password) to file? y/n: ")
|
|
if dec == "y":
|
|
with open(config_filename, "w") as f:
|
|
json.dump(config, f, indent=2)
|
|
|
|
print(f"Stored new config to {config_filename}")
|
|
|
|
config["password"] = password
|
|
return config
|
|
|
|
|
|
def find_mmauthtoken_firefox(host):
|
|
appdata_dir = pathlib.Path(os.environ["APPDATA"])
|
|
profiles_dir = appdata_dir / "Mozilla/Firefox/Profiles"
|
|
cookie_files = profiles_dir.rglob("cookies.sqlite")
|
|
|
|
all_tokens = []
|
|
for cookie_file in cookie_files:
|
|
print(f"Opening {cookie_file}")
|
|
connection = sqlite3.connect(str(cookie_file))
|
|
cursor = connection.cursor()
|
|
rows = cursor.execute("SELECT host, value FROM moz_cookies WHERE name = 'MMAUTHTOKEN'").fetchall()
|
|
all_tokens.extend(rows)
|
|
|
|
all_tokens = [token for token in all_tokens if host in token[0]]
|
|
|
|
print(f"Found {len(all_tokens)} token for {host}")
|
|
for token in all_tokens:
|
|
print(f"{token[0]}: {token[1]}")
|
|
|
|
if len(all_tokens) > 1:
|
|
print("Using first token!")
|
|
|
|
if len(all_tokens):
|
|
return all_tokens[0][1]
|
|
else:
|
|
return None
|
|
|
|
|
|
def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Download Mattermost channel history to local JSON files.",
|
|
)
|
|
parser.add_argument(
|
|
"--config",
|
|
default="config.json",
|
|
help="Path to config file (default: config.json).",
|
|
)
|
|
parser.add_argument(
|
|
"--team",
|
|
help="Team name for non-interactive export (e.g. esr-daily).",
|
|
)
|
|
parser.add_argument(
|
|
"--channel",
|
|
help="Channel name(s), comma-separated, or 'all' for every channel in the team.",
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
help="Output directory (overrides config output_base and default date-based path).",
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
args = parse_args()
|
|
non_interactive = bool(args.team or args.channel)
|
|
if non_interactive and not (args.team and args.channel):
|
|
print("Non-interactive mode requires both --team and --channel.", file=sys.stderr)
|
|
raise SystemExit(2)
|
|
|
|
config_filename = args.config
|
|
config = get_config_from_json(config_filename)
|
|
config = complete_config(config, config_filename, interactive=not non_interactive)
|
|
|
|
incremental = config.get("incremental", bool(config.get("last_export_at")))
|
|
if args.output:
|
|
output_base = args.output
|
|
after = config.get("after")
|
|
before = config.get("before")
|
|
merge = False
|
|
elif incremental and config.get("last_export_at"):
|
|
print(f"Incremental update from {config['last_export_at']}")
|
|
output_base = config.get("output_base") or ("results/" + date.today().strftime("%Y%m%d"))
|
|
after = config["last_export_at"]
|
|
before = config.get("before")
|
|
merge = True
|
|
else:
|
|
output_base = "results/" + date.today().strftime("%Y%m%d")
|
|
after = config.get("after")
|
|
before = config.get("before")
|
|
merge = False
|
|
|
|
print(f"Storing downloaded data in {output_base}")
|
|
export_started_at = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
try:
|
|
d = connect(
|
|
config["host"],
|
|
config.get("token", None),
|
|
config.get("username", None),
|
|
config.get("password", None),
|
|
)
|
|
apply_throttle(d, config)
|
|
except Exception as e:
|
|
if non_interactive:
|
|
raise SystemExit(f"Authentication failed: {e}") from e
|
|
# If token-based authentication failed, drop the token and ask
|
|
# the user to rerun the script so that fresh credentials can be
|
|
# obtained or entered.
|
|
if config.get("token"):
|
|
print("Token-based authentication failed, removing stored token.")
|
|
config.pop("token", None)
|
|
save_config(config, config_filename)
|
|
print("Please restart the script to reconfigure authentication.")
|
|
raise SystemExit(1) from e
|
|
raise
|
|
user_id_to_name, my_user_id, user_profiles = get_users(d)
|
|
save_users_json(output_base, user_profiles)
|
|
if non_interactive:
|
|
team = find_team(d, my_user_id, args.team)
|
|
print(f"Selected team {team['name']}")
|
|
channels = get_channels_for_team(d, team, my_user_id, user_id_to_name)
|
|
print(f"Found {len(channels)} channels")
|
|
channels = find_channels(channels, args.channel)
|
|
print("Selected channel(s):", ", ".join(channel["display_name"] for channel in channels))
|
|
else:
|
|
team = select_team(d, my_user_id)
|
|
channels = select_channel(d, team, my_user_id, user_id_to_name)
|
|
if after:
|
|
channels = filter_channels_by_last_post(channels, after, output_base, merge=merge)
|
|
for i_channel, channel in enumerate(channels):
|
|
print(f"Start export of channel {i_channel + 1}/{len(channels)}")
|
|
export_channel(d, channel, user_id_to_name, output_base, config["download_files"],
|
|
before, after, team=team, merge=merge)
|
|
|
|
config["last_export_at"] = export_started_at
|
|
config["output_base"] = output_base
|
|
config["incremental"] = True
|
|
save_config(config, config_filename)
|
|
print(f"Updated last_export_at to {export_started_at}")
|
|
print("Finished export")
|