add user index
This commit is contained in:
+22
-3
@@ -270,7 +270,7 @@ def get_token_via_gitlab(host: str, gitlab_url: str, username: str, password: st
|
|||||||
return session.cookies.get("MMAUTHTOKEN")
|
return session.cookies.get("MMAUTHTOKEN")
|
||||||
|
|
||||||
|
|
||||||
def get_users(d: Driver) -> Tuple[Dict[str, str], str]:
|
def get_users(d: Driver) -> Tuple[Dict[str, str], str, List[dict]]:
|
||||||
my_user = d.users.get_user("me")
|
my_user = d.users.get_user("me")
|
||||||
my_username = my_user["username"]
|
my_username = my_user["username"]
|
||||||
my_user_id = my_user["id"]
|
my_user_id = my_user["id"]
|
||||||
@@ -278,6 +278,7 @@ def get_users(d: Driver) -> Tuple[Dict[str, str], str]:
|
|||||||
|
|
||||||
# Get all usernames as we want to use those instead of the user ids
|
# Get all usernames as we want to use those instead of the user ids
|
||||||
user_id_to_name = {}
|
user_id_to_name = {}
|
||||||
|
user_profiles = []
|
||||||
page = 0
|
page = 0
|
||||||
print("Downloading all user information... ", end="")
|
print("Downloading all user information... ", end="")
|
||||||
while True:
|
while True:
|
||||||
@@ -286,10 +287,27 @@ def get_users(d: Driver) -> Tuple[Dict[str, str], str]:
|
|||||||
break
|
break
|
||||||
for user in users_resp:
|
for user in users_resp:
|
||||||
user_id_to_name[user["id"]] = user["username"]
|
user_id_to_name[user["id"]] = user["username"]
|
||||||
|
user_profiles.append(
|
||||||
|
{
|
||||||
|
"username": user.get("username") or "",
|
||||||
|
"first_name": user.get("first_name") or "",
|
||||||
|
"last_name": user.get("last_name") or "",
|
||||||
|
"nickname": user.get("nickname") or "",
|
||||||
|
}
|
||||||
|
)
|
||||||
page += 1
|
page += 1
|
||||||
print(f"Found {len(user_id_to_name)} users!")
|
print(f"Found {len(user_id_to_name)} users!")
|
||||||
|
|
||||||
return user_id_to_name, my_user_id
|
return user_id_to_name, my_user_id, user_profiles
|
||||||
|
|
||||||
|
|
||||||
|
def save_users_json(output_base: str, user_profiles: List[dict]) -> str:
|
||||||
|
output_path = pathlib.Path(output_base) / "users.json"
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(user_profiles, f, indent=2, ensure_ascii=False)
|
||||||
|
print(f"Saved user directory to '{output_path}'")
|
||||||
|
return str(output_path)
|
||||||
|
|
||||||
|
|
||||||
def select_team(d: Driver, my_user_id: str) -> str:
|
def select_team(d: Driver, my_user_id: str) -> str:
|
||||||
@@ -731,7 +749,8 @@ if __name__ == '__main__':
|
|||||||
print("Please restart the script to reconfigure authentication.")
|
print("Please restart the script to reconfigure authentication.")
|
||||||
raise SystemExit(1) from e
|
raise SystemExit(1) from e
|
||||||
raise
|
raise
|
||||||
user_id_to_name, my_user_id = get_users(d)
|
user_id_to_name, my_user_id, user_profiles = get_users(d)
|
||||||
|
save_users_json(output_base, user_profiles)
|
||||||
team = select_team(d, my_user_id)
|
team = select_team(d, my_user_id)
|
||||||
channels = select_channel(d, team, my_user_id, user_id_to_name)
|
channels = select_channel(d, team, my_user_id, user_id_to_name)
|
||||||
for i_channel, channel in enumerate(channels):
|
for i_channel, channel in enumerate(channels):
|
||||||
|
|||||||
@@ -12,11 +12,8 @@ description: |
|
|||||||
# Mattermost Search (MCP)
|
# Mattermost Search (MCP)
|
||||||
|
|
||||||
Search local Mattermost JSON exports produced by `mattermost-dl.py`. **Always use MCP
|
Search local Mattermost JSON exports produced by `mattermost-dl.py`. **Always use MCP
|
||||||
tools** (`mm_search`, `mm_search_index_status`) as the default path.
|
tools** (`mm_search`, `mm_resolve_user`, `mm_search_users`, `mm_user_directory_refresh`,
|
||||||
|
`mm_search_index_status`) as the default path.
|
||||||
**Ежедневные планы и отчёты (#plan / #report):** [everyday.md](everyday.md)
|
|
||||||
|
|
||||||
Username: `sergey.marinkevich`
|
|
||||||
|
|
||||||
## Mandatory rules
|
## Mandatory rules
|
||||||
|
|
||||||
@@ -24,6 +21,11 @@ Username: `sergey.marinkevich`
|
|||||||
|
|
||||||
Не делать самостоятельно, пока пользователь явно не попросил:
|
Не делать самостоятельно, пока пользователь явно не попросил:
|
||||||
|
|
||||||
|
Всё про ежедневные планы или отчёты смотри: [everyday.md](everyday.md). Там и формат описан, и каналы.
|
||||||
|
|
||||||
|
My username: `sergey.marinkevich`. Resolve others via `mm_resolve_user(ref="Цисык")` →
|
||||||
|
`vladislav.tsisyk` (needs user directory — see below).
|
||||||
|
|
||||||
| Обход | Примеры |
|
| Обход | Примеры |
|
||||||
|---|---|
|
|---|---|
|
||||||
| Индекс | `no_index=true`, `regex=true` (сканирует JSON), прямое чтение/grep по `results/` |
|
| Индекс | `no_index=true`, `regex=true` (сканирует JSON), прямое чтение/grep по `results/` |
|
||||||
@@ -49,10 +51,11 @@ Server env (set in `mcp.json` `cwd` / `env`):
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `MM_SEARCH_ROOT` | `results` |
|
| `MM_SEARCH_ROOT` | `results` |
|
||||||
| `MM_SEARCH_DB` | `mattermost_index.sqlite` |
|
| `MM_SEARCH_DB` | `mattermost_index.sqlite` |
|
||||||
|
| `MM_CONFIG_PATH` | `config.json` (for `mm_user_directory_refresh(from_api=true)`) |
|
||||||
|
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
1. `mm_search_index_status()` — confirm `db_exists`, `root_exists`, `json_files`.
|
1. `mm_search_index_status()` without paramters — confirm `db_exists`, `root_exists`, `json_files`.
|
||||||
2. Pick **search_mode** (see table below).
|
2. Pick **search_mode** (see table below).
|
||||||
3. `mm_search(query=…, …)` with filters; keep `limit` small (10–50) unless the user needs all hits.
|
3. `mm_search(query=…, …)` with filters; keep `limit` small (10–50) unless the user needs all hits.
|
||||||
4. Read `matches[].post.message`; for attachments also `matches[].attachment`.
|
4. Read `matches[].post.message`; for attachments also `matches[].attachment`.
|
||||||
@@ -62,7 +65,14 @@ Server env (set in `mcp.json` `cwd` / `env`):
|
|||||||
| Need | Tool |
|
| Need | Tool |
|
||||||
|---|---|
|
|---|---|
|
||||||
| Search messages / attachments | `mm_search(query, …)` |
|
| Search messages / attachments | `mm_search(query, …)` |
|
||||||
| Check index & export layout | `mm_search_index_status(db?, root?)` |
|
| Resolve display name → username | `mm_resolve_user(ref)` |
|
||||||
|
| Search users by name | `mm_search_users(query)` |
|
||||||
|
| Refresh user directory | `mm_user_directory_refresh(from_api=true)` |
|
||||||
|
| Check index & export layout | `mm_search_index_status(db, root)` |
|
||||||
|
|
||||||
|
User directory is stored in the sqlite DB (`users` table). Sources: `users.json` in export
|
||||||
|
root (written by `mattermost-dl.py`), Mattermost API (`from_api=true`), and distinct
|
||||||
|
usernames from indexed posts.
|
||||||
|
|
||||||
## Choose search_mode
|
## Choose search_mode
|
||||||
|
|
||||||
@@ -136,6 +146,20 @@ mm_search(query="deploy rollback", user="alice", after="2026-01-01", limit=20)
|
|||||||
mm_search(query="incident", channel="alerts", limit=30)
|
mm_search(query="incident", channel="alerts", limit=30)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Resolve user, then search their posts**
|
||||||
|
|
||||||
|
```
|
||||||
|
mm_user_directory_refresh(from_api=true)
|
||||||
|
mm_resolve_user(ref="Цисык")
|
||||||
|
mm_search(query="#report", user="vladislav.tsisyk", channel="System/HW", limit=30)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check somebody report** (when username is known)
|
||||||
|
|
||||||
|
```
|
||||||
|
mm_search(query="#report", user="ivan.pavlov", channel="System/HW", limit=30)
|
||||||
|
```
|
||||||
|
|
||||||
**Find attachment by name**
|
**Find attachment by name**
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -158,7 +182,10 @@ mm_search(query="standup", output="human", show_file=true, limit=15)
|
|||||||
|
|
||||||
- `file_name` / `file_content` + `regex=true` → error.
|
- `file_name` / `file_content` + `regex=true` → error.
|
||||||
- Attachment search without index → error; run `mm_index.py` first.
|
- Attachment search without index → error; run `mm_index.py` first.
|
||||||
- `channel` / `user` / `team` are **exact** matches, not substring.
|
- `channel` / `user` / `team` are **exact** matches, not substring — resolve display names
|
||||||
|
with `mm_resolve_user` first.
|
||||||
|
- User directory empty → run `mm_user_directory_refresh(from_api=true)` or re-export with
|
||||||
|
`mattermost-dl.py` (writes `users.json`).
|
||||||
- Token budget: prefer `limit`, narrow `after`/`before`, then widen if empty.
|
- Token budget: prefer `limit`, narrow `after`/`before`, then widen if empty.
|
||||||
- If `count=0` with index: broaden `query`, drop filters, suggest `mm_index.py` — **не**
|
- If `count=0` with index: broaden `query`, drop filters, suggest `mm_index.py` — **не**
|
||||||
переключаться на `no_index` / `regex` / CLI без явного разрешения.
|
переключаться на `no_index` / `regex` / CLI без явного разрешения.
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import os
|
|||||||
import sqlite3
|
import sqlite3
|
||||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||||
|
|
||||||
|
import mm_users
|
||||||
|
|
||||||
|
|
||||||
def iter_result_files(root: str) -> Iterable[str]:
|
def iter_result_files(root: str) -> Iterable[str]:
|
||||||
for dirpath, _dirnames, filenames in os.walk(root):
|
for dirpath, _dirnames, filenames in os.walk(root):
|
||||||
@@ -105,6 +107,7 @@ def connect_db(db_path: str) -> sqlite3.Connection:
|
|||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
mm_users.ensure_users_schema(conn)
|
||||||
return conn
|
return conn
|
||||||
|
|
||||||
|
|
||||||
@@ -307,6 +310,10 @@ def main() -> int:
|
|||||||
rows_upserted += upserted
|
rows_upserted += upserted
|
||||||
rows_skipped += skipped
|
rows_skipped += skipped
|
||||||
|
|
||||||
|
users_json = mm_users.find_users_json(args.root)
|
||||||
|
if users_json:
|
||||||
|
mm_users.upsert_users(conn, mm_users.load_users_json(users_json))
|
||||||
|
|
||||||
total_posts = conn.execute("SELECT COUNT(*) AS c FROM posts").fetchone()["c"]
|
total_posts = conn.execute("SELECT COUNT(*) AS c FROM posts").fetchone()["c"]
|
||||||
total_sources = conn.execute("SELECT COUNT(*) AS c FROM sources").fetchone()["c"]
|
total_sources = conn.execute("SELECT COUNT(*) AS c FROM sources").fetchone()["c"]
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|||||||
+55
-4
@@ -7,6 +7,8 @@ import re
|
|||||||
import sqlite3
|
import sqlite3
|
||||||
from typing import Any, Dict, Iterable, List, Optional
|
from typing import Any, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
import mm_users
|
||||||
|
|
||||||
|
|
||||||
def iter_result_files(root: str) -> Iterable[str]:
|
def iter_result_files(root: str) -> Iterable[str]:
|
||||||
for dirpath, _dirnames, filenames in os.walk(root):
|
for dirpath, _dirnames, filenames in os.walk(root):
|
||||||
@@ -102,7 +104,7 @@ def search_index(
|
|||||||
JOIN posts_fts fts ON fts.post_id = p.post_id
|
JOIN posts_fts fts ON fts.post_id = p.post_id
|
||||||
WHERE fts.message MATCH ?
|
WHERE fts.message MATCH ?
|
||||||
{where_filters}
|
{where_filters}
|
||||||
ORDER BY p.created ASC
|
ORDER BY p.created DESC
|
||||||
"""
|
"""
|
||||||
params = [fts_q] + params
|
params = [fts_q] + params
|
||||||
else:
|
else:
|
||||||
@@ -115,7 +117,7 @@ def search_index(
|
|||||||
FROM posts p
|
FROM posts p
|
||||||
WHERE {cmp_col} LIKE ?
|
WHERE {cmp_col} LIKE ?
|
||||||
{where_filters}
|
{where_filters}
|
||||||
ORDER BY p.created ASC
|
ORDER BY p.created DESC
|
||||||
"""
|
"""
|
||||||
params = [f"%{needle}%"] + params
|
params = [f"%{needle}%"] + params
|
||||||
|
|
||||||
@@ -218,7 +220,7 @@ def search_attachments_index(
|
|||||||
JOIN posts p ON p.post_id = a.post_id
|
JOIN posts p ON p.post_id = a.post_id
|
||||||
WHERE aft.{fts_column} MATCH ?
|
WHERE aft.{fts_column} MATCH ?
|
||||||
{where_filters}
|
{where_filters}
|
||||||
ORDER BY p.created ASC
|
ORDER BY p.created DESC
|
||||||
"""
|
"""
|
||||||
|
|
||||||
params = [fts_q] + params
|
params = [fts_q] + params
|
||||||
@@ -359,7 +361,7 @@ def main(argv: Optional[List[str]] = None) -> int:
|
|||||||
ap = argparse.ArgumentParser(
|
ap = argparse.ArgumentParser(
|
||||||
description="Search through Mattermost export JSON files produced by mattermost-dl.py.",
|
description="Search through Mattermost export JSON files produced by mattermost-dl.py.",
|
||||||
)
|
)
|
||||||
ap.add_argument("query", help="Search string or regex (see --regex).")
|
ap.add_argument("query", nargs="?", help="Search string or regex (see --regex).")
|
||||||
ap.add_argument(
|
ap.add_argument(
|
||||||
"-r",
|
"-r",
|
||||||
"--root",
|
"--root",
|
||||||
@@ -407,9 +409,58 @@ def main(argv: Optional[List[str]] = None) -> int:
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Include source filename and per-file index in human output.",
|
help="Include source filename and per-file index in human output.",
|
||||||
)
|
)
|
||||||
|
user_group = ap.add_mutually_exclusive_group()
|
||||||
|
user_group.add_argument(
|
||||||
|
"--resolve-user",
|
||||||
|
metavar="REF",
|
||||||
|
help="Resolve display name / FIO / nickname to username (uses sqlite user directory).",
|
||||||
|
)
|
||||||
|
user_group.add_argument(
|
||||||
|
"--search-users",
|
||||||
|
metavar="QUERY",
|
||||||
|
help="Search users by name or username substring.",
|
||||||
|
)
|
||||||
|
ap.add_argument(
|
||||||
|
"--refresh-users",
|
||||||
|
action="store_true",
|
||||||
|
help="Refresh user directory from users.json, optional API, and post usernames.",
|
||||||
|
)
|
||||||
|
ap.add_argument(
|
||||||
|
"--from-api",
|
||||||
|
action="store_true",
|
||||||
|
help="With --refresh-users, also fetch profiles from Mattermost API (config.json).",
|
||||||
|
)
|
||||||
|
ap.add_argument(
|
||||||
|
"--config",
|
||||||
|
default=os.environ.get("MM_CONFIG_PATH", "config.json"),
|
||||||
|
help="Mattermost config path for --from-api (default: MM_CONFIG_PATH or config.json).",
|
||||||
|
)
|
||||||
|
|
||||||
args = ap.parse_args(argv)
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
if args.refresh_users:
|
||||||
|
result = mm_users.refresh_user_directory(
|
||||||
|
args.db,
|
||||||
|
args.root,
|
||||||
|
config_path=args.config,
|
||||||
|
from_api=args.from_api,
|
||||||
|
)
|
||||||
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if args.resolve_user:
|
||||||
|
result = mm_users.resolve_user(args.db, args.resolve_user)
|
||||||
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if args.search_users:
|
||||||
|
matches = mm_users.search_users(args.db, args.search_users, limit=args.limit or 10)
|
||||||
|
print(json.dumps(matches, ensure_ascii=False, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if not args.query:
|
||||||
|
ap.error("query is required unless using --resolve-user, --search-users, or --refresh-users")
|
||||||
|
|
||||||
# Default ignore-case for non-regex search unless explicitly disabled
|
# Default ignore-case for non-regex search unless explicitly disabled
|
||||||
ignore_case = args.ignore_case or not args.regex
|
ignore_case = args.ignore_case or not args.regex
|
||||||
matcher = make_matcher(args.query, args.regex, ignore_case)
|
matcher = make_matcher(args.query, args.regex, ignore_case)
|
||||||
|
|||||||
+82
-4
@@ -28,6 +28,7 @@ from typing import Any, Literal, Optional
|
|||||||
from fastmcp import FastMCP
|
from fastmcp import FastMCP
|
||||||
|
|
||||||
import mm_search as mm_search_lib
|
import mm_search as mm_search_lib
|
||||||
|
import mm_users
|
||||||
|
|
||||||
mcp = FastMCP("mattermost-search")
|
mcp = FastMCP("mattermost-search")
|
||||||
|
|
||||||
@@ -61,7 +62,7 @@ def _collect_matches(
|
|||||||
regex: bool,
|
regex: bool,
|
||||||
search_mode: SearchMode,
|
search_mode: SearchMode,
|
||||||
ignore_case: bool,
|
ignore_case: bool,
|
||||||
limit: int,
|
limit: int = 10,
|
||||||
db: str,
|
db: str,
|
||||||
no_index: bool,
|
no_index: bool,
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
@@ -167,7 +168,7 @@ def mm_search(
|
|||||||
regex: bool = False,
|
regex: bool = False,
|
||||||
search_mode: SearchMode = "message",
|
search_mode: SearchMode = "message",
|
||||||
ignore_case: bool = None,
|
ignore_case: bool = None,
|
||||||
limit: int = None,
|
limit: int = 10,
|
||||||
db: str = None,
|
db: str = None,
|
||||||
no_index: bool = False,
|
no_index: bool = False,
|
||||||
output: Literal["json", "human"] = "json",
|
output: Literal["json", "human"] = "json",
|
||||||
@@ -188,7 +189,7 @@ def mm_search(
|
|||||||
regex: Treat query as regular expression (message scan only).
|
regex: Treat query as regular expression (message scan only).
|
||||||
search_mode: "message" (default), "file_name", or "file_content".
|
search_mode: "message" (default), "file_name", or "file_content".
|
||||||
ignore_case: Case-insensitive search. Default: True for substring, False for regex.
|
ignore_case: Case-insensitive search. Default: True for substring, False for regex.
|
||||||
limit: Maximum number of matches to return.
|
limit: Maximum number of matches to return (default: 10).
|
||||||
db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite").
|
db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite").
|
||||||
no_index: Do not use sqlite index even if DB exists.
|
no_index: Do not use sqlite index even if DB exists.
|
||||||
output: "json" returns match objects; "human" returns CLI-style text lines.
|
output: "json" returns match objects; "human" returns CLI-style text lines.
|
||||||
@@ -234,9 +235,73 @@ def mm_search(
|
|||||||
|
|
||||||
|
|
||||||
@mcp.tool
|
@mcp.tool
|
||||||
def mm_search_index_status(
|
def mm_resolve_user(
|
||||||
|
ref: str,
|
||||||
db: str = None,
|
db: str = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Resolve display name, FIO, nickname, or username to Mattermost username.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
ref: User reference — username, first/last name, nickname, or FIO
|
||||||
|
(e.g. "Цисык", "vladislav.tsisyk").
|
||||||
|
db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite").
|
||||||
|
"""
|
||||||
|
resolved_db = db or _env_db()
|
||||||
|
return mm_users.resolve_user(resolved_db, ref)
|
||||||
|
|
||||||
|
|
||||||
|
@mcp.tool
|
||||||
|
def mm_search_users(
|
||||||
|
query: str,
|
||||||
|
limit: int = 10,
|
||||||
|
db: str = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Search users in the local directory by name or username.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Substring to match against username, first/last name, nickname, full name.
|
||||||
|
limit: Maximum matches (default: 10).
|
||||||
|
db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite").
|
||||||
|
"""
|
||||||
|
resolved_db = db or _env_db()
|
||||||
|
matches = mm_users.search_users(resolved_db, query, limit=limit)
|
||||||
|
return {
|
||||||
|
"query": query,
|
||||||
|
"count": len(matches),
|
||||||
|
"db": resolved_db,
|
||||||
|
"matches": matches,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@mcp.tool
|
||||||
|
def mm_user_directory_refresh(
|
||||||
root: str = None,
|
root: str = None,
|
||||||
|
db: str = None,
|
||||||
|
from_api: bool = False,
|
||||||
|
config_path: str = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Rebuild the local user directory from users.json, Mattermost API, and post usernames.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
root: Export root (default: MM_SEARCH_ROOT or "results").
|
||||||
|
db: SQLite index path (default: MM_SEARCH_DB).
|
||||||
|
from_api: Also fetch profiles from Mattermost API (needs config.json with token).
|
||||||
|
config_path: Path to Mattermost config (default: MM_CONFIG_PATH or "config.json").
|
||||||
|
"""
|
||||||
|
resolved_root = root or _env_root()
|
||||||
|
resolved_db = db or _env_db()
|
||||||
|
return mm_users.refresh_user_directory(
|
||||||
|
resolved_db,
|
||||||
|
resolved_root,
|
||||||
|
config_path=config_path,
|
||||||
|
from_api=from_api,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@mcp.tool
|
||||||
|
def mm_search_index_status(
|
||||||
|
db: Optional[str] = None,
|
||||||
|
root: Optional[str] = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Report whether the sqlite index and export root are available."""
|
"""Report whether the sqlite index and export root are available."""
|
||||||
resolved_db = db or _env_db()
|
resolved_db = db or _env_db()
|
||||||
@@ -248,12 +313,25 @@ def mm_search_index_status(
|
|||||||
if root_exists:
|
if root_exists:
|
||||||
json_count = sum(1 for _ in mm_search_lib.iter_result_files(resolved_root))
|
json_count = sum(1 for _ in mm_search_lib.iter_result_files(resolved_root))
|
||||||
|
|
||||||
|
user_count = 0
|
||||||
|
if db_exists:
|
||||||
|
try:
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
conn = sqlite3.connect(resolved_db)
|
||||||
|
mm_users.ensure_users_schema(conn)
|
||||||
|
user_count = conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]
|
||||||
|
conn.close()
|
||||||
|
except sqlite3.Error:
|
||||||
|
user_count = None
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"db": resolved_db,
|
"db": resolved_db,
|
||||||
"db_exists": db_exists,
|
"db_exists": db_exists,
|
||||||
"root": resolved_root,
|
"root": resolved_root,
|
||||||
"root_exists": root_exists,
|
"root_exists": root_exists,
|
||||||
"json_files": json_count,
|
"json_files": json_count,
|
||||||
|
"users": user_count,
|
||||||
"index_required_for": ["file_name", "file_content"],
|
"index_required_for": ["file_name", "file_content"],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+318
@@ -0,0 +1,318 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""User directory for Mattermost export search (resolve display names to usernames)."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime as dt
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def _norm(s: Optional[str]) -> str:
|
||||||
|
return (s or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _full_name(first_name: str, last_name: str) -> str:
|
||||||
|
parts = [_norm(first_name), _norm(last_name)]
|
||||||
|
return " ".join(p for p in parts if p)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_users_schema(conn: sqlite3.Connection) -> None:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS users (
|
||||||
|
username TEXT PRIMARY KEY,
|
||||||
|
first_name TEXT NOT NULL DEFAULT '',
|
||||||
|
last_name TEXT NOT NULL DEFAULT '',
|
||||||
|
nickname TEXT NOT NULL DEFAULT '',
|
||||||
|
full_name TEXT NOT NULL DEFAULT '',
|
||||||
|
updated_at TEXT NOT NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_users_full_name ON users(full_name)")
|
||||||
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_users_first_name ON users(first_name)")
|
||||||
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_users_last_name ON users(last_name)")
|
||||||
|
|
||||||
|
|
||||||
|
def profile_from_api_user(user: Dict[str, Any]) -> Dict[str, str]:
|
||||||
|
first_name = _norm(user.get("first_name"))
|
||||||
|
last_name = _norm(user.get("last_name"))
|
||||||
|
nickname = _norm(user.get("nickname"))
|
||||||
|
username = _norm(user.get("username"))
|
||||||
|
return {
|
||||||
|
"username": username,
|
||||||
|
"first_name": first_name,
|
||||||
|
"last_name": last_name,
|
||||||
|
"nickname": nickname,
|
||||||
|
"full_name": _full_name(first_name, last_name),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def load_users_json(path: str) -> List[Dict[str, str]]:
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
if not isinstance(data, list):
|
||||||
|
raise ValueError(f"users.json must be a JSON array: {path}")
|
||||||
|
out: List[Dict[str, str]] = []
|
||||||
|
for item in data:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
username = _norm(item.get("username"))
|
||||||
|
if not username:
|
||||||
|
continue
|
||||||
|
first_name = _norm(item.get("first_name"))
|
||||||
|
last_name = _norm(item.get("last_name"))
|
||||||
|
nickname = _norm(item.get("nickname"))
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"username": username,
|
||||||
|
"first_name": first_name,
|
||||||
|
"last_name": last_name,
|
||||||
|
"nickname": nickname,
|
||||||
|
"full_name": _full_name(first_name, last_name),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def find_users_json(root: str) -> Optional[str]:
|
||||||
|
direct = os.path.join(root, "users.json")
|
||||||
|
if os.path.isfile(direct):
|
||||||
|
return direct
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_users_from_api(config_path: str) -> List[Dict[str, str]]:
|
||||||
|
with open(config_path, encoding="utf-8") as f:
|
||||||
|
config = json.load(f)
|
||||||
|
host = _norm(config.get("host"))
|
||||||
|
token = _norm(config.get("token"))
|
||||||
|
if not host or not token:
|
||||||
|
raise ValueError(f"config must contain host and token: {config_path}")
|
||||||
|
|
||||||
|
from mattermostdriver import Driver
|
||||||
|
|
||||||
|
d = Driver(
|
||||||
|
{
|
||||||
|
"url": host,
|
||||||
|
"token": token,
|
||||||
|
"scheme": "https",
|
||||||
|
"port": 443,
|
||||||
|
"verify": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
d.login()
|
||||||
|
|
||||||
|
profiles: List[Dict[str, str]] = []
|
||||||
|
page = 0
|
||||||
|
while True:
|
||||||
|
users_resp = d.users.get_users(params={"per_page": 200, "page": page})
|
||||||
|
if not users_resp:
|
||||||
|
break
|
||||||
|
for user in users_resp:
|
||||||
|
profile = profile_from_api_user(user)
|
||||||
|
if profile["username"]:
|
||||||
|
profiles.append(profile)
|
||||||
|
page += 1
|
||||||
|
return profiles
|
||||||
|
|
||||||
|
|
||||||
|
def usernames_from_posts(conn: sqlite3.Connection) -> List[str]:
|
||||||
|
rows = conn.execute("SELECT DISTINCT username FROM posts WHERE username != '' ORDER BY username").fetchall()
|
||||||
|
return [str(row[0]) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_users(conn: sqlite3.Connection, profiles: Iterable[Dict[str, str]]) -> int:
|
||||||
|
ensure_users_schema(conn)
|
||||||
|
now = dt.datetime.now(dt.timezone.utc).isoformat()
|
||||||
|
count = 0
|
||||||
|
for profile in profiles:
|
||||||
|
username = _norm(profile.get("username"))
|
||||||
|
if not username:
|
||||||
|
continue
|
||||||
|
first_name = _norm(profile.get("first_name"))
|
||||||
|
last_name = _norm(profile.get("last_name"))
|
||||||
|
nickname = _norm(profile.get("nickname"))
|
||||||
|
full_name = _norm(profile.get("full_name")) or _full_name(first_name, last_name)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO users (username, first_name, last_name, nickname, full_name, updated_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(username) DO UPDATE SET
|
||||||
|
first_name = excluded.first_name,
|
||||||
|
last_name = excluded.last_name,
|
||||||
|
nickname = excluded.nickname,
|
||||||
|
full_name = excluded.full_name,
|
||||||
|
updated_at = excluded.updated_at
|
||||||
|
""",
|
||||||
|
(username, first_name, last_name, nickname, full_name, now),
|
||||||
|
)
|
||||||
|
count += 1
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_user_directory(
|
||||||
|
db_path: str,
|
||||||
|
root: str,
|
||||||
|
*,
|
||||||
|
config_path: Optional[str] = None,
|
||||||
|
from_api: bool = False,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
ensure_users_schema(conn)
|
||||||
|
sources: List[str] = []
|
||||||
|
profiles: List[Dict[str, str]] = []
|
||||||
|
|
||||||
|
users_json = find_users_json(root)
|
||||||
|
if users_json:
|
||||||
|
profiles.extend(load_users_json(users_json))
|
||||||
|
sources.append(f"users.json:{users_json}")
|
||||||
|
|
||||||
|
if from_api:
|
||||||
|
cfg = config_path or os.environ.get("MM_CONFIG_PATH", "config.json")
|
||||||
|
if not os.path.isfile(cfg):
|
||||||
|
raise FileNotFoundError(f"Mattermost config not found: {cfg}")
|
||||||
|
profiles.extend(fetch_users_from_api(cfg))
|
||||||
|
sources.append(f"api:{cfg}")
|
||||||
|
|
||||||
|
with conn:
|
||||||
|
upserted_named = upsert_users(conn, profiles) if profiles else 0
|
||||||
|
post_only = 0
|
||||||
|
known = {p["username"] for p in profiles}
|
||||||
|
for username in usernames_from_posts(conn):
|
||||||
|
if username in known:
|
||||||
|
continue
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO users (username, first_name, last_name, nickname, full_name, updated_at)
|
||||||
|
VALUES (?, '', '', '', '', ?)
|
||||||
|
ON CONFLICT(username) DO NOTHING
|
||||||
|
""",
|
||||||
|
(username, dt.datetime.now(dt.timezone.utc).isoformat()),
|
||||||
|
)
|
||||||
|
post_only += 1
|
||||||
|
sources.append("posts:distinct_usernames")
|
||||||
|
|
||||||
|
total = conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]
|
||||||
|
conn.close()
|
||||||
|
return {
|
||||||
|
"db": db_path,
|
||||||
|
"root": root,
|
||||||
|
"sources": sources,
|
||||||
|
"upserted_profiles": upserted_named,
|
||||||
|
"added_from_posts": post_only,
|
||||||
|
"total_users": total,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _user_row_to_dict(row: sqlite3.Row) -> Dict[str, Any]:
|
||||||
|
first_name = row["first_name"] or ""
|
||||||
|
last_name = row["last_name"] or ""
|
||||||
|
nickname = row["nickname"] or ""
|
||||||
|
full_name = row["full_name"] or _full_name(first_name, last_name)
|
||||||
|
display = full_name or nickname or row["username"]
|
||||||
|
return {
|
||||||
|
"username": row["username"],
|
||||||
|
"first_name": first_name,
|
||||||
|
"last_name": last_name,
|
||||||
|
"nickname": nickname,
|
||||||
|
"name": display,
|
||||||
|
"full_name": full_name,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _match_score(ref: str, row: sqlite3.Row) -> int:
|
||||||
|
ref_l = ref.casefold()
|
||||||
|
username = row["username"] or ""
|
||||||
|
first_name = row["first_name"] or ""
|
||||||
|
last_name = row["last_name"] or ""
|
||||||
|
nickname = row["nickname"] or ""
|
||||||
|
full_name = row["full_name"] or _full_name(first_name, last_name)
|
||||||
|
|
||||||
|
if username.casefold() == ref_l:
|
||||||
|
return 100
|
||||||
|
for field in (first_name, last_name, nickname, full_name):
|
||||||
|
if field and field.casefold() == ref_l:
|
||||||
|
return 90
|
||||||
|
for field in (first_name, last_name, nickname, full_name, username):
|
||||||
|
if field and ref_l in field.casefold():
|
||||||
|
return 70
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _field_contains(field: str, ref: str) -> bool:
|
||||||
|
return ref.casefold() in (field or "").casefold()
|
||||||
|
|
||||||
|
|
||||||
|
def search_users(
|
||||||
|
db_path: str,
|
||||||
|
query: str,
|
||||||
|
*,
|
||||||
|
limit: int = 10,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
ref = _norm(query)
|
||||||
|
if not ref:
|
||||||
|
return []
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
ensure_users_schema(conn)
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT username, first_name, last_name, nickname, full_name FROM users"
|
||||||
|
).fetchall()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
matched = [
|
||||||
|
row
|
||||||
|
for row in rows
|
||||||
|
if any(
|
||||||
|
_field_contains(field, ref)
|
||||||
|
for field in (
|
||||||
|
row["username"],
|
||||||
|
row["first_name"],
|
||||||
|
row["last_name"],
|
||||||
|
row["nickname"],
|
||||||
|
row["full_name"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
scored = [(_match_score(ref, row), _user_row_to_dict(row)) for row in matched]
|
||||||
|
scored.sort(key=lambda item: (-item[0], item[1]["username"]))
|
||||||
|
return [item[1] for item in scored[:limit]]
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_user(db_path: str, ref: str) -> Dict[str, Any]:
|
||||||
|
ref = _norm(ref)
|
||||||
|
if not ref:
|
||||||
|
raise ValueError("ref must be non-empty")
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
ensure_users_schema(conn)
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT username, first_name, last_name, nickname, full_name FROM users"
|
||||||
|
).fetchall()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
best = [row for row in rows if _match_score(ref, row) > 0]
|
||||||
|
best.sort(key=lambda row: (-_match_score(ref, row), row["username"]))
|
||||||
|
|
||||||
|
if not best:
|
||||||
|
return {"found": False, "ref": ref, "matches": []}
|
||||||
|
|
||||||
|
top = _match_score(ref, best[0])
|
||||||
|
top_matches = [row for row in best if _match_score(ref, row) == top]
|
||||||
|
result_matches = [_user_row_to_dict(row) for row in top_matches[:10]]
|
||||||
|
|
||||||
|
if len(result_matches) == 1:
|
||||||
|
user = result_matches[0]
|
||||||
|
return {"found": True, "ref": ref, "username": user["username"], **user}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"found": True,
|
||||||
|
"ambiguous": True,
|
||||||
|
"ref": ref,
|
||||||
|
"matches": result_matches,
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user