From b83d4f8b9b14fbd16e88f88b1b6e156dc380eca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9C=D0=B0=D1=80?= =?UTF-8?q?=D0=B8=D0=BD=D0=BA=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 5 Jun 2026 17:19:12 +0700 Subject: [PATCH] add user index --- mattermost-dl.py | 25 ++- mattermost-search/SKILL.md | 43 ++++- mm_index.py | 7 + mm_search.py | 59 ++++++- mm_search_mcp.py | 86 +++++++++- mm_users.py | 318 +++++++++++++++++++++++++++++++++++++ 6 files changed, 519 insertions(+), 19 deletions(-) create mode 100644 mm_users.py diff --git a/mattermost-dl.py b/mattermost-dl.py index 47c05fc..1de62b4 100644 --- a/mattermost-dl.py +++ b/mattermost-dl.py @@ -270,7 +270,7 @@ def get_token_via_gitlab(host: str, gitlab_url: str, username: str, password: st return session.cookies.get("MMAUTHTOKEN") -def get_users(d: Driver) -> Tuple[Dict[str, str], str]: +def get_users(d: Driver) -> Tuple[Dict[str, str], str, List[dict]]: my_user = d.users.get_user("me") my_username = my_user["username"] my_user_id = my_user["id"] @@ -278,6 +278,7 @@ def get_users(d: Driver) -> Tuple[Dict[str, str], str]: # Get all usernames as we want to use those instead of the user ids user_id_to_name = {} + user_profiles = [] page = 0 print("Downloading all user information... ", end="") while True: @@ -286,10 +287,27 @@ def get_users(d: Driver) -> Tuple[Dict[str, str], str]: break for user in users_resp: user_id_to_name[user["id"]] = user["username"] + user_profiles.append( + { + "username": user.get("username") or "", + "first_name": user.get("first_name") or "", + "last_name": user.get("last_name") or "", + "nickname": user.get("nickname") or "", + } + ) page += 1 print(f"Found {len(user_id_to_name)} users!") - return user_id_to_name, my_user_id + return user_id_to_name, my_user_id, user_profiles + + +def save_users_json(output_base: str, user_profiles: List[dict]) -> str: + output_path = pathlib.Path(output_base) / "users.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(user_profiles, f, indent=2, ensure_ascii=False) + print(f"Saved user directory to '{output_path}'") + return str(output_path) def select_team(d: Driver, my_user_id: str) -> str: @@ -731,7 +749,8 @@ if __name__ == '__main__': print("Please restart the script to reconfigure authentication.") raise SystemExit(1) from e raise - user_id_to_name, my_user_id = get_users(d) + user_id_to_name, my_user_id, user_profiles = get_users(d) + save_users_json(output_base, user_profiles) team = select_team(d, my_user_id) channels = select_channel(d, team, my_user_id, user_id_to_name) for i_channel, channel in enumerate(channels): diff --git a/mattermost-search/SKILL.md b/mattermost-search/SKILL.md index be1e1ce..c043968 100644 --- a/mattermost-search/SKILL.md +++ b/mattermost-search/SKILL.md @@ -12,11 +12,8 @@ description: | # Mattermost Search (MCP) Search local Mattermost JSON exports produced by `mattermost-dl.py`. **Always use MCP -tools** (`mm_search`, `mm_search_index_status`) as the default path. - -**Ежедневные планы и отчёты (#plan / #report):** [everyday.md](everyday.md) - -Username: `sergey.marinkevich` +tools** (`mm_search`, `mm_resolve_user`, `mm_search_users`, `mm_user_directory_refresh`, +`mm_search_index_status`) as the default path. ## Mandatory rules @@ -24,6 +21,11 @@ Username: `sergey.marinkevich` Не делать самостоятельно, пока пользователь явно не попросил: +Всё про ежедневные планы или отчёты смотри: [everyday.md](everyday.md). Там и формат описан, и каналы. + +My username: `sergey.marinkevich`. Resolve others via `mm_resolve_user(ref="Цисык")` → +`vladislav.tsisyk` (needs user directory — see below). + | Обход | Примеры | |---|---| | Индекс | `no_index=true`, `regex=true` (сканирует JSON), прямое чтение/grep по `results/` | @@ -49,10 +51,11 @@ Server env (set in `mcp.json` `cwd` / `env`): |---|---| | `MM_SEARCH_ROOT` | `results` | | `MM_SEARCH_DB` | `mattermost_index.sqlite` | +| `MM_CONFIG_PATH` | `config.json` (for `mm_user_directory_refresh(from_api=true)`) | ## Quick start -1. `mm_search_index_status()` — confirm `db_exists`, `root_exists`, `json_files`. +1. `mm_search_index_status()` without paramters — confirm `db_exists`, `root_exists`, `json_files`. 2. Pick **search_mode** (see table below). 3. `mm_search(query=…, …)` with filters; keep `limit` small (10–50) unless the user needs all hits. 4. Read `matches[].post.message`; for attachments also `matches[].attachment`. @@ -62,7 +65,14 @@ Server env (set in `mcp.json` `cwd` / `env`): | Need | Tool | |---|---| | Search messages / attachments | `mm_search(query, …)` | -| Check index & export layout | `mm_search_index_status(db?, root?)` | +| Resolve display name → username | `mm_resolve_user(ref)` | +| Search users by name | `mm_search_users(query)` | +| Refresh user directory | `mm_user_directory_refresh(from_api=true)` | +| Check index & export layout | `mm_search_index_status(db, root)` | + +User directory is stored in the sqlite DB (`users` table). Sources: `users.json` in export +root (written by `mattermost-dl.py`), Mattermost API (`from_api=true`), and distinct +usernames from indexed posts. ## Choose search_mode @@ -136,6 +146,20 @@ mm_search(query="deploy rollback", user="alice", after="2026-01-01", limit=20) mm_search(query="incident", channel="alerts", limit=30) ``` +**Resolve user, then search their posts** + +``` +mm_user_directory_refresh(from_api=true) +mm_resolve_user(ref="Цисык") +mm_search(query="#report", user="vladislav.tsisyk", channel="System/HW", limit=30) +``` + +**Check somebody report** (when username is known) + +``` +mm_search(query="#report", user="ivan.pavlov", channel="System/HW", limit=30) +``` + **Find attachment by name** ``` @@ -158,7 +182,10 @@ mm_search(query="standup", output="human", show_file=true, limit=15) - `file_name` / `file_content` + `regex=true` → error. - Attachment search without index → error; run `mm_index.py` first. -- `channel` / `user` / `team` are **exact** matches, not substring. +- `channel` / `user` / `team` are **exact** matches, not substring — resolve display names + with `mm_resolve_user` first. +- User directory empty → run `mm_user_directory_refresh(from_api=true)` or re-export with + `mattermost-dl.py` (writes `users.json`). - Token budget: prefer `limit`, narrow `after`/`before`, then widen if empty. - If `count=0` with index: broaden `query`, drop filters, suggest `mm_index.py` — **не** переключаться на `no_index` / `regex` / CLI без явного разрешения. diff --git a/mm_index.py b/mm_index.py index fdd6f34..2b5fdbe 100644 --- a/mm_index.py +++ b/mm_index.py @@ -6,6 +6,8 @@ import os import sqlite3 from typing import Any, Dict, Iterable, List, Optional, Tuple +import mm_users + def iter_result_files(root: str) -> Iterable[str]: for dirpath, _dirnames, filenames in os.walk(root): @@ -105,6 +107,7 @@ def connect_db(db_path: str) -> sqlite3.Connection: ) """ ) + mm_users.ensure_users_schema(conn) return conn @@ -307,6 +310,10 @@ def main() -> int: rows_upserted += upserted rows_skipped += skipped + users_json = mm_users.find_users_json(args.root) + if users_json: + mm_users.upsert_users(conn, mm_users.load_users_json(users_json)) + total_posts = conn.execute("SELECT COUNT(*) AS c FROM posts").fetchone()["c"] total_sources = conn.execute("SELECT COUNT(*) AS c FROM sources").fetchone()["c"] conn.close() diff --git a/mm_search.py b/mm_search.py index 16207b0..61a3dcf 100644 --- a/mm_search.py +++ b/mm_search.py @@ -7,6 +7,8 @@ import re import sqlite3 from typing import Any, Dict, Iterable, List, Optional +import mm_users + def iter_result_files(root: str) -> Iterable[str]: for dirpath, _dirnames, filenames in os.walk(root): @@ -102,7 +104,7 @@ def search_index( JOIN posts_fts fts ON fts.post_id = p.post_id WHERE fts.message MATCH ? {where_filters} - ORDER BY p.created ASC + ORDER BY p.created DESC """ params = [fts_q] + params else: @@ -115,7 +117,7 @@ def search_index( FROM posts p WHERE {cmp_col} LIKE ? {where_filters} - ORDER BY p.created ASC + ORDER BY p.created DESC """ params = [f"%{needle}%"] + params @@ -218,7 +220,7 @@ def search_attachments_index( JOIN posts p ON p.post_id = a.post_id WHERE aft.{fts_column} MATCH ? {where_filters} - ORDER BY p.created ASC + ORDER BY p.created DESC """ params = [fts_q] + params @@ -359,7 +361,7 @@ def main(argv: Optional[List[str]] = None) -> int: ap = argparse.ArgumentParser( description="Search through Mattermost export JSON files produced by mattermost-dl.py.", ) - ap.add_argument("query", help="Search string or regex (see --regex).") + ap.add_argument("query", nargs="?", help="Search string or regex (see --regex).") ap.add_argument( "-r", "--root", @@ -407,9 +409,58 @@ def main(argv: Optional[List[str]] = None) -> int: action="store_true", help="Include source filename and per-file index in human output.", ) + user_group = ap.add_mutually_exclusive_group() + user_group.add_argument( + "--resolve-user", + metavar="REF", + help="Resolve display name / FIO / nickname to username (uses sqlite user directory).", + ) + user_group.add_argument( + "--search-users", + metavar="QUERY", + help="Search users by name or username substring.", + ) + ap.add_argument( + "--refresh-users", + action="store_true", + help="Refresh user directory from users.json, optional API, and post usernames.", + ) + ap.add_argument( + "--from-api", + action="store_true", + help="With --refresh-users, also fetch profiles from Mattermost API (config.json).", + ) + ap.add_argument( + "--config", + default=os.environ.get("MM_CONFIG_PATH", "config.json"), + help="Mattermost config path for --from-api (default: MM_CONFIG_PATH or config.json).", + ) args = ap.parse_args(argv) + if args.refresh_users: + result = mm_users.refresh_user_directory( + args.db, + args.root, + config_path=args.config, + from_api=args.from_api, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + if args.resolve_user: + result = mm_users.resolve_user(args.db, args.resolve_user) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + if args.search_users: + matches = mm_users.search_users(args.db, args.search_users, limit=args.limit or 10) + print(json.dumps(matches, ensure_ascii=False, indent=2)) + return 0 + + if not args.query: + ap.error("query is required unless using --resolve-user, --search-users, or --refresh-users") + # Default ignore-case for non-regex search unless explicitly disabled ignore_case = args.ignore_case or not args.regex matcher = make_matcher(args.query, args.regex, ignore_case) diff --git a/mm_search_mcp.py b/mm_search_mcp.py index f1ba1ea..e351695 100644 --- a/mm_search_mcp.py +++ b/mm_search_mcp.py @@ -28,6 +28,7 @@ from typing import Any, Literal, Optional from fastmcp import FastMCP import mm_search as mm_search_lib +import mm_users mcp = FastMCP("mattermost-search") @@ -61,7 +62,7 @@ def _collect_matches( regex: bool, search_mode: SearchMode, ignore_case: bool, - limit: int, + limit: int = 10, db: str, no_index: bool, ) -> list[dict[str, Any]]: @@ -167,7 +168,7 @@ def mm_search( regex: bool = False, search_mode: SearchMode = "message", ignore_case: bool = None, - limit: int = None, + limit: int = 10, db: str = None, no_index: bool = False, output: Literal["json", "human"] = "json", @@ -188,7 +189,7 @@ def mm_search( regex: Treat query as regular expression (message scan only). search_mode: "message" (default), "file_name", or "file_content". ignore_case: Case-insensitive search. Default: True for substring, False for regex. - limit: Maximum number of matches to return. + limit: Maximum number of matches to return (default: 10). db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite"). no_index: Do not use sqlite index even if DB exists. output: "json" returns match objects; "human" returns CLI-style text lines. @@ -234,9 +235,73 @@ def mm_search( @mcp.tool -def mm_search_index_status( +def mm_resolve_user( + ref: str, db: str = None, +) -> dict[str, Any]: + """Resolve display name, FIO, nickname, or username to Mattermost username. + + Args: + ref: User reference — username, first/last name, nickname, or FIO + (e.g. "Цисык", "vladislav.tsisyk"). + db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite"). + """ + resolved_db = db or _env_db() + return mm_users.resolve_user(resolved_db, ref) + + +@mcp.tool +def mm_search_users( + query: str, + limit: int = 10, + db: str = None, +) -> dict[str, Any]: + """Search users in the local directory by name or username. + + Args: + query: Substring to match against username, first/last name, nickname, full name. + limit: Maximum matches (default: 10). + db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite"). + """ + resolved_db = db or _env_db() + matches = mm_users.search_users(resolved_db, query, limit=limit) + return { + "query": query, + "count": len(matches), + "db": resolved_db, + "matches": matches, + } + + +@mcp.tool +def mm_user_directory_refresh( root: str = None, + db: str = None, + from_api: bool = False, + config_path: str = None, +) -> dict[str, Any]: + """Rebuild the local user directory from users.json, Mattermost API, and post usernames. + + Args: + root: Export root (default: MM_SEARCH_ROOT or "results"). + db: SQLite index path (default: MM_SEARCH_DB). + from_api: Also fetch profiles from Mattermost API (needs config.json with token). + config_path: Path to Mattermost config (default: MM_CONFIG_PATH or "config.json"). + """ + resolved_root = root or _env_root() + resolved_db = db or _env_db() + return mm_users.refresh_user_directory( + resolved_db, + resolved_root, + config_path=config_path, + from_api=from_api, + ) + + +@mcp.tool +def mm_search_index_status( + db: Optional[str] = None, + root: Optional[str] = None, ) -> dict[str, Any]: """Report whether the sqlite index and export root are available.""" resolved_db = db or _env_db() @@ -248,12 +313,25 @@ def mm_search_index_status( if root_exists: json_count = sum(1 for _ in mm_search_lib.iter_result_files(resolved_root)) + user_count = 0 + if db_exists: + try: + import sqlite3 + + conn = sqlite3.connect(resolved_db) + mm_users.ensure_users_schema(conn) + user_count = conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] + conn.close() + except sqlite3.Error: + user_count = None + return { "db": resolved_db, "db_exists": db_exists, "root": resolved_root, "root_exists": root_exists, "json_files": json_count, + "users": user_count, "index_required_for": ["file_name", "file_content"], } diff --git a/mm_users.py b/mm_users.py new file mode 100644 index 0000000..a371e9c --- /dev/null +++ b/mm_users.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""User directory for Mattermost export search (resolve display names to usernames).""" +from __future__ import annotations + +import datetime as dt +import json +import os +import sqlite3 +from typing import Any, Dict, Iterable, List, Optional + + +def _norm(s: Optional[str]) -> str: + return (s or "").strip() + + +def _full_name(first_name: str, last_name: str) -> str: + parts = [_norm(first_name), _norm(last_name)] + return " ".join(p for p in parts if p) + + +def ensure_users_schema(conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS users ( + username TEXT PRIMARY KEY, + first_name TEXT NOT NULL DEFAULT '', + last_name TEXT NOT NULL DEFAULT '', + nickname TEXT NOT NULL DEFAULT '', + full_name TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_users_full_name ON users(full_name)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_users_first_name ON users(first_name)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_users_last_name ON users(last_name)") + + +def profile_from_api_user(user: Dict[str, Any]) -> Dict[str, str]: + first_name = _norm(user.get("first_name")) + last_name = _norm(user.get("last_name")) + nickname = _norm(user.get("nickname")) + username = _norm(user.get("username")) + return { + "username": username, + "first_name": first_name, + "last_name": last_name, + "nickname": nickname, + "full_name": _full_name(first_name, last_name), + } + + +def load_users_json(path: str) -> List[Dict[str, str]]: + with open(path, encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, list): + raise ValueError(f"users.json must be a JSON array: {path}") + out: List[Dict[str, str]] = [] + for item in data: + if not isinstance(item, dict): + continue + username = _norm(item.get("username")) + if not username: + continue + first_name = _norm(item.get("first_name")) + last_name = _norm(item.get("last_name")) + nickname = _norm(item.get("nickname")) + out.append( + { + "username": username, + "first_name": first_name, + "last_name": last_name, + "nickname": nickname, + "full_name": _full_name(first_name, last_name), + } + ) + return out + + +def find_users_json(root: str) -> Optional[str]: + direct = os.path.join(root, "users.json") + if os.path.isfile(direct): + return direct + return None + + +def fetch_users_from_api(config_path: str) -> List[Dict[str, str]]: + with open(config_path, encoding="utf-8") as f: + config = json.load(f) + host = _norm(config.get("host")) + token = _norm(config.get("token")) + if not host or not token: + raise ValueError(f"config must contain host and token: {config_path}") + + from mattermostdriver import Driver + + d = Driver( + { + "url": host, + "token": token, + "scheme": "https", + "port": 443, + "verify": True, + } + ) + d.login() + + profiles: List[Dict[str, str]] = [] + page = 0 + while True: + users_resp = d.users.get_users(params={"per_page": 200, "page": page}) + if not users_resp: + break + for user in users_resp: + profile = profile_from_api_user(user) + if profile["username"]: + profiles.append(profile) + page += 1 + return profiles + + +def usernames_from_posts(conn: sqlite3.Connection) -> List[str]: + rows = conn.execute("SELECT DISTINCT username FROM posts WHERE username != '' ORDER BY username").fetchall() + return [str(row[0]) for row in rows] + + +def upsert_users(conn: sqlite3.Connection, profiles: Iterable[Dict[str, str]]) -> int: + ensure_users_schema(conn) + now = dt.datetime.now(dt.timezone.utc).isoformat() + count = 0 + for profile in profiles: + username = _norm(profile.get("username")) + if not username: + continue + first_name = _norm(profile.get("first_name")) + last_name = _norm(profile.get("last_name")) + nickname = _norm(profile.get("nickname")) + full_name = _norm(profile.get("full_name")) or _full_name(first_name, last_name) + conn.execute( + """ + INSERT INTO users (username, first_name, last_name, nickname, full_name, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(username) DO UPDATE SET + first_name = excluded.first_name, + last_name = excluded.last_name, + nickname = excluded.nickname, + full_name = excluded.full_name, + updated_at = excluded.updated_at + """, + (username, first_name, last_name, nickname, full_name, now), + ) + count += 1 + return count + + +def refresh_user_directory( + db_path: str, + root: str, + *, + config_path: Optional[str] = None, + from_api: bool = False, +) -> Dict[str, Any]: + conn = sqlite3.connect(db_path) + ensure_users_schema(conn) + sources: List[str] = [] + profiles: List[Dict[str, str]] = [] + + users_json = find_users_json(root) + if users_json: + profiles.extend(load_users_json(users_json)) + sources.append(f"users.json:{users_json}") + + if from_api: + cfg = config_path or os.environ.get("MM_CONFIG_PATH", "config.json") + if not os.path.isfile(cfg): + raise FileNotFoundError(f"Mattermost config not found: {cfg}") + profiles.extend(fetch_users_from_api(cfg)) + sources.append(f"api:{cfg}") + + with conn: + upserted_named = upsert_users(conn, profiles) if profiles else 0 + post_only = 0 + known = {p["username"] for p in profiles} + for username in usernames_from_posts(conn): + if username in known: + continue + conn.execute( + """ + INSERT INTO users (username, first_name, last_name, nickname, full_name, updated_at) + VALUES (?, '', '', '', '', ?) + ON CONFLICT(username) DO NOTHING + """, + (username, dt.datetime.now(dt.timezone.utc).isoformat()), + ) + post_only += 1 + sources.append("posts:distinct_usernames") + + total = conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] + conn.close() + return { + "db": db_path, + "root": root, + "sources": sources, + "upserted_profiles": upserted_named, + "added_from_posts": post_only, + "total_users": total, + } + + +def _user_row_to_dict(row: sqlite3.Row) -> Dict[str, Any]: + first_name = row["first_name"] or "" + last_name = row["last_name"] or "" + nickname = row["nickname"] or "" + full_name = row["full_name"] or _full_name(first_name, last_name) + display = full_name or nickname or row["username"] + return { + "username": row["username"], + "first_name": first_name, + "last_name": last_name, + "nickname": nickname, + "name": display, + "full_name": full_name, + } + + +def _match_score(ref: str, row: sqlite3.Row) -> int: + ref_l = ref.casefold() + username = row["username"] or "" + first_name = row["first_name"] or "" + last_name = row["last_name"] or "" + nickname = row["nickname"] or "" + full_name = row["full_name"] or _full_name(first_name, last_name) + + if username.casefold() == ref_l: + return 100 + for field in (first_name, last_name, nickname, full_name): + if field and field.casefold() == ref_l: + return 90 + for field in (first_name, last_name, nickname, full_name, username): + if field and ref_l in field.casefold(): + return 70 + return 0 + + +def _field_contains(field: str, ref: str) -> bool: + return ref.casefold() in (field or "").casefold() + + +def search_users( + db_path: str, + query: str, + *, + limit: int = 10, +) -> List[Dict[str, Any]]: + ref = _norm(query) + if not ref: + return [] + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + ensure_users_schema(conn) + rows = conn.execute( + "SELECT username, first_name, last_name, nickname, full_name FROM users" + ).fetchall() + conn.close() + + matched = [ + row + for row in rows + if any( + _field_contains(field, ref) + for field in ( + row["username"], + row["first_name"], + row["last_name"], + row["nickname"], + row["full_name"], + ) + ) + ] + scored = [(_match_score(ref, row), _user_row_to_dict(row)) for row in matched] + scored.sort(key=lambda item: (-item[0], item[1]["username"])) + return [item[1] for item in scored[:limit]] + + +def resolve_user(db_path: str, ref: str) -> Dict[str, Any]: + ref = _norm(ref) + if not ref: + raise ValueError("ref must be non-empty") + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + ensure_users_schema(conn) + rows = conn.execute( + "SELECT username, first_name, last_name, nickname, full_name FROM users" + ).fetchall() + conn.close() + + best = [row for row in rows if _match_score(ref, row) > 0] + best.sort(key=lambda row: (-_match_score(ref, row), row["username"])) + + if not best: + return {"found": False, "ref": ref, "matches": []} + + top = _match_score(ref, best[0]) + top_matches = [row for row in best if _match_score(ref, row) == top] + result_matches = [_user_row_to_dict(row) for row in top_matches[:10]] + + if len(result_matches) == 1: + user = result_matches[0] + return {"found": True, "ref": ref, "username": user["username"], **user} + + return { + "found": True, + "ambiguous": True, + "ref": ref, + "matches": result_matches, + }