Сергей Маринкевич
6f8648f7fa
Avoid per-agent stdio contention on the sqlite index, sort FTS by rank with created tie-break, and add mm_search_tag for dated #plan/#report substring search. Co-authored-by: Cursor <cursoragent@cursor.com>
329 lines
10 KiB
Python
329 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""User directory for Mattermost export search (resolve display names to usernames)."""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from typing import Any, Dict, Iterable, List, Optional
|
|
|
|
|
|
def _norm(s: Optional[str]) -> str:
|
|
return (s or "").strip()
|
|
|
|
|
|
def _full_name(first_name: str, last_name: str) -> str:
|
|
parts = [_norm(first_name), _norm(last_name)]
|
|
return " ".join(p for p in parts if p)
|
|
|
|
|
|
def ensure_users_schema(conn: sqlite3.Connection) -> None:
|
|
conn.execute(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
username TEXT PRIMARY KEY,
|
|
first_name TEXT NOT NULL DEFAULT '',
|
|
last_name TEXT NOT NULL DEFAULT '',
|
|
nickname TEXT NOT NULL DEFAULT '',
|
|
full_name TEXT NOT NULL DEFAULT '',
|
|
updated_at TEXT NOT NULL
|
|
)
|
|
"""
|
|
)
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_users_full_name ON users(full_name)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_users_first_name ON users(first_name)")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_users_last_name ON users(last_name)")
|
|
|
|
|
|
def profile_from_api_user(user: Dict[str, Any]) -> Dict[str, str]:
|
|
first_name = _norm(user.get("first_name"))
|
|
last_name = _norm(user.get("last_name"))
|
|
nickname = _norm(user.get("nickname"))
|
|
username = _norm(user.get("username"))
|
|
return {
|
|
"username": username,
|
|
"first_name": first_name,
|
|
"last_name": last_name,
|
|
"nickname": nickname,
|
|
"full_name": _full_name(first_name, last_name),
|
|
}
|
|
|
|
|
|
def load_users_json(path: str) -> List[Dict[str, str]]:
|
|
with open(path, encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
if not isinstance(data, list):
|
|
raise ValueError(f"users.json must be a JSON array: {path}")
|
|
out: List[Dict[str, str]] = []
|
|
for item in data:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
username = _norm(item.get("username"))
|
|
if not username:
|
|
continue
|
|
first_name = _norm(item.get("first_name"))
|
|
last_name = _norm(item.get("last_name"))
|
|
nickname = _norm(item.get("nickname"))
|
|
out.append(
|
|
{
|
|
"username": username,
|
|
"first_name": first_name,
|
|
"last_name": last_name,
|
|
"nickname": nickname,
|
|
"full_name": _full_name(first_name, last_name),
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def find_users_json(root: str) -> Optional[str]:
|
|
direct = os.path.join(root, "users.json")
|
|
if os.path.isfile(direct):
|
|
return direct
|
|
return None
|
|
|
|
|
|
def fetch_users_from_api(config_path: str) -> List[Dict[str, str]]:
|
|
with open(config_path, encoding="utf-8") as f:
|
|
config = json.load(f)
|
|
host = _norm(config.get("host"))
|
|
token = _norm(config.get("token"))
|
|
if not host or not token:
|
|
raise ValueError(f"config must contain host and token: {config_path}")
|
|
|
|
from mattermostdriver import Driver
|
|
|
|
d = Driver(
|
|
{
|
|
"url": host,
|
|
"token": token,
|
|
"scheme": "https",
|
|
"port": 443,
|
|
"verify": True,
|
|
}
|
|
)
|
|
d.login()
|
|
|
|
profiles: List[Dict[str, str]] = []
|
|
page = 0
|
|
while True:
|
|
users_resp = d.users.get_users(params={"per_page": 200, "page": page})
|
|
if not users_resp:
|
|
break
|
|
for user in users_resp:
|
|
profile = profile_from_api_user(user)
|
|
if profile["username"]:
|
|
profiles.append(profile)
|
|
page += 1
|
|
return profiles
|
|
|
|
|
|
def usernames_from_posts(conn: sqlite3.Connection) -> List[str]:
|
|
rows = conn.execute("SELECT DISTINCT username FROM posts WHERE username != '' ORDER BY username").fetchall()
|
|
return [str(row[0]) for row in rows]
|
|
|
|
|
|
def upsert_users(conn: sqlite3.Connection, profiles: Iterable[Dict[str, str]]) -> int:
|
|
ensure_users_schema(conn)
|
|
now = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
count = 0
|
|
for profile in profiles:
|
|
username = _norm(profile.get("username"))
|
|
if not username:
|
|
continue
|
|
first_name = _norm(profile.get("first_name"))
|
|
last_name = _norm(profile.get("last_name"))
|
|
nickname = _norm(profile.get("nickname"))
|
|
full_name = _norm(profile.get("full_name")) or _full_name(first_name, last_name)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO users (username, first_name, last_name, nickname, full_name, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(username) DO UPDATE SET
|
|
first_name = excluded.first_name,
|
|
last_name = excluded.last_name,
|
|
nickname = excluded.nickname,
|
|
full_name = excluded.full_name,
|
|
updated_at = excluded.updated_at
|
|
""",
|
|
(username, first_name, last_name, nickname, full_name, now),
|
|
)
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def refresh_user_directory(
|
|
db_path: str,
|
|
root: str,
|
|
*,
|
|
config_path: Optional[str] = None,
|
|
from_api: bool = False,
|
|
) -> Dict[str, Any]:
|
|
from mm_search import open_db
|
|
|
|
conn = open_db(db_path, readonly=False, timeout=30.0)
|
|
ensure_users_schema(conn)
|
|
sources: List[str] = []
|
|
profiles: List[Dict[str, str]] = []
|
|
|
|
users_json = find_users_json(root)
|
|
if users_json:
|
|
profiles.extend(load_users_json(users_json))
|
|
sources.append(f"users.json:{users_json}")
|
|
|
|
if from_api:
|
|
cfg = config_path or os.environ.get("MM_CONFIG_PATH", "config.json")
|
|
if not os.path.isfile(cfg):
|
|
raise FileNotFoundError(f"Mattermost config not found: {cfg}")
|
|
profiles.extend(fetch_users_from_api(cfg))
|
|
sources.append(f"api:{cfg}")
|
|
|
|
with conn:
|
|
upserted_named = upsert_users(conn, profiles) if profiles else 0
|
|
post_only = 0
|
|
known = {p["username"] for p in profiles}
|
|
for username in usernames_from_posts(conn):
|
|
if username in known:
|
|
continue
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO users (username, first_name, last_name, nickname, full_name, updated_at)
|
|
VALUES (?, '', '', '', '', ?)
|
|
ON CONFLICT(username) DO NOTHING
|
|
""",
|
|
(username, dt.datetime.now(dt.timezone.utc).isoformat()),
|
|
)
|
|
post_only += 1
|
|
sources.append("posts:distinct_usernames")
|
|
|
|
total = conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]
|
|
conn.close()
|
|
return {
|
|
"db": db_path,
|
|
"root": root,
|
|
"sources": sources,
|
|
"upserted_profiles": upserted_named,
|
|
"added_from_posts": post_only,
|
|
"total_users": total,
|
|
}
|
|
|
|
|
|
def _user_row_to_dict(row: sqlite3.Row) -> Dict[str, Any]:
|
|
first_name = row["first_name"] or ""
|
|
last_name = row["last_name"] or ""
|
|
nickname = row["nickname"] or ""
|
|
full_name = row["full_name"] or _full_name(first_name, last_name)
|
|
display = full_name or nickname or row["username"]
|
|
return {
|
|
"username": row["username"],
|
|
"first_name": first_name,
|
|
"last_name": last_name,
|
|
"nickname": nickname,
|
|
"name": display,
|
|
"full_name": full_name,
|
|
}
|
|
|
|
|
|
def _match_score(ref: str, row: sqlite3.Row) -> int:
|
|
ref_l = ref.casefold()
|
|
username = row["username"] or ""
|
|
first_name = row["first_name"] or ""
|
|
last_name = row["last_name"] or ""
|
|
nickname = row["nickname"] or ""
|
|
full_name = row["full_name"] or _full_name(first_name, last_name)
|
|
|
|
if username.casefold() == ref_l:
|
|
return 100
|
|
for field in (first_name, last_name, nickname, full_name):
|
|
if field and field.casefold() == ref_l:
|
|
return 90
|
|
for field in (first_name, last_name, nickname, full_name, username):
|
|
if field and ref_l in field.casefold():
|
|
return 70
|
|
return 0
|
|
|
|
|
|
def _field_contains(field: str, ref: str) -> bool:
|
|
return ref.casefold() in (field or "").casefold()
|
|
|
|
|
|
def search_users(
|
|
db_path: str,
|
|
query: str,
|
|
*,
|
|
limit: int = 10,
|
|
) -> List[Dict[str, Any]]:
|
|
ref = _norm(query)
|
|
if not ref:
|
|
return []
|
|
|
|
from mm_search import open_db
|
|
|
|
conn = open_db(db_path, readonly=True)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT username, first_name, last_name, nickname, full_name FROM users"
|
|
).fetchall()
|
|
except sqlite3.Error:
|
|
conn.close()
|
|
return []
|
|
conn.close()
|
|
|
|
matched = [
|
|
row
|
|
for row in rows
|
|
if any(
|
|
_field_contains(field, ref)
|
|
for field in (
|
|
row["username"],
|
|
row["first_name"],
|
|
row["last_name"],
|
|
row["nickname"],
|
|
row["full_name"],
|
|
)
|
|
)
|
|
]
|
|
scored = [(_match_score(ref, row), _user_row_to_dict(row)) for row in matched]
|
|
scored.sort(key=lambda item: (-item[0], item[1]["username"]))
|
|
return [item[1] for item in scored[:limit]]
|
|
|
|
|
|
def resolve_user(db_path: str, ref: str) -> Dict[str, Any]:
|
|
ref = _norm(ref)
|
|
if not ref:
|
|
raise ValueError("ref must be non-empty")
|
|
|
|
from mm_search import open_db
|
|
|
|
conn = open_db(db_path, readonly=True)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT username, first_name, last_name, nickname, full_name FROM users"
|
|
).fetchall()
|
|
except sqlite3.Error:
|
|
conn.close()
|
|
return {"found": False, "ref": ref, "matches": [], "error": "users table missing; run mm_user_directory_refresh"}
|
|
conn.close()
|
|
|
|
best = [row for row in rows if _match_score(ref, row) > 0]
|
|
best.sort(key=lambda row: (-_match_score(ref, row), row["username"]))
|
|
|
|
if not best:
|
|
return {"found": False, "ref": ref, "matches": []}
|
|
|
|
top = _match_score(ref, best[0])
|
|
top_matches = [row for row in best if _match_score(ref, row) == top]
|
|
result_matches = [_user_row_to_dict(row) for row in top_matches[:10]]
|
|
|
|
if len(result_matches) == 1:
|
|
user = result_matches[0]
|
|
return {"found": True, "ref": ref, "username": user["username"], **user}
|
|
|
|
return {
|
|
"found": True,
|
|
"ambiguous": True,
|
|
"ref": ref,
|
|
"matches": result_matches,
|
|
}
|