From 99dc9f173df957624afdd7791cabe586e89dc818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9C=D0=B0=D1=80?= =?UTF-8?q?=D0=B8=D0=BD=D0=BA=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 5 Jun 2026 15:09:07 +0700 Subject: [PATCH] add mcp --- mattermost-dl.py | 77 +++++++++++ mattermost-search/SKILL.md | 172 ++++++++++++++++++++++++ mm_search_mcp.py | 262 +++++++++++++++++++++++++++++++++++++ 3 files changed, 511 insertions(+) create mode 100644 mattermost-search/SKILL.md create mode 100644 mm_search_mcp.py diff --git a/mattermost-dl.py b/mattermost-dl.py index 7d15f05..54f736f 100644 --- a/mattermost-dl.py +++ b/mattermost-dl.py @@ -1,6 +1,8 @@ import os import pickle +import random import sqlite3 +import time from datetime import datetime, date, timezone from typing import Tuple, Dict, List, Optional import getpass @@ -15,6 +17,80 @@ import json GITLAB_URL_DEFAULT = "https://mmgit.eltex-co.ru" COOKIE_FILE_DEFAULT = "cookies.pkl" +THROTTLE_DEFAULTS = { + "min_delay": 0.4, + "max_delay": 1.8, + "burst_pause_every": 12, + "burst_pause_min": 3.0, + "burst_pause_max": 8.0, + "file_delay_multiplier": 1.5, +} + + +class RequestThrottler: + """Paces API calls with jitter and occasional longer pauses.""" + + def __init__(self, **opts): + self.min_delay = opts["min_delay"] + self.max_delay = opts["max_delay"] + self.burst_pause_every = opts["burst_pause_every"] + self.burst_pause_min = opts["burst_pause_min"] + self.burst_pause_max = opts["burst_pause_max"] + self.file_delay_multiplier = opts["file_delay_multiplier"] + self._request_count = 0 + self._last_request_at = 0.0 + + def wait(self, endpoint: str = "") -> None: + self._request_count += 1 + if self.burst_pause_every and self._request_count % self.burst_pause_every == 0: + delay = random.uniform(self.burst_pause_min, self.burst_pause_max) + else: + delay = random.uniform(self.min_delay, self.max_delay) + + if endpoint.startswith("/files/"): + delay *= self.file_delay_multiplier + + elapsed = time.monotonic() - self._last_request_at + if elapsed < delay: + time.sleep(delay - elapsed) + self._last_request_at = time.monotonic() + + +def _parse_throttle_config(config: dict) -> Optional[dict]: + throttle = config.get("throttle") + if not throttle: + return None + if throttle is True: + return dict(THROTTLE_DEFAULTS) + if isinstance(throttle, dict): + if not throttle.get("enabled", True): + return None + opts = dict(THROTTLE_DEFAULTS) + opts.update({k: v for k, v in throttle.items() if k != "enabled"}) + return opts + return None + + +def apply_throttle(d: Driver, config: dict) -> None: + opts = _parse_throttle_config(config) + if not opts: + return + + throttler = RequestThrottler(**opts) + client = d.client + original_make_request = client.make_request + + def throttled_make_request(method, endpoint, options=None, params=None, data=None, files=None, basepath=None): + throttler.wait(endpoint) + return original_make_request(method, endpoint, options, params, data, files, basepath) + + client.make_request = throttled_make_request + print( + "Request throttling enabled " + f"(delay {opts['min_delay']}-{opts['max_delay']}s, " + f"pause every {opts['burst_pause_every']} requests)" + ) + def connect(host: str, login_token: str = None, username: str = None, password: str = None) -> Driver: d = Driver({ @@ -568,6 +644,7 @@ if __name__ == '__main__': config.get("username", None), config.get("password", None), ) + apply_throttle(d, config) except Exception as e: # If token-based authentication failed, drop the token and ask # the user to rerun the script so that fresh credentials can be diff --git a/mattermost-search/SKILL.md b/mattermost-search/SKILL.md new file mode 100644 index 0000000..f0c9a88 --- /dev/null +++ b/mattermost-search/SKILL.md @@ -0,0 +1,172 @@ +--- +name: mattermost-search +description: | + Search Mattermost export dumps via mattermost-search MCP (mm_search_mcp.py): + message text, attachment names, attachment text content, filters by user/channel/team/date. + Use when the user asks to find messages, posts, or files in Mattermost exports, + mattermost-dl results, or the local sqlite index. Requires MCP server mattermost-search. + Index bypass (no_index, regex, CLI, grep) needs explicit user permission. +--- + +# Mattermost Search (MCP) + +Search local Mattermost JSON exports produced by `mattermost-dl.py`. **Always use MCP +tools** (`mm_search`, `mm_search_index_status`) as the default path. + +## Mandatory rules + +**Поиск в обход индекса и MCP — только с явного разрешения пользователя.** + +Не делать самостоятельно, пока пользователь явно не попросил: + +| Обход | Примеры | +|---|---| +| Индекс | `no_index=true`, `regex=true` (сканирует JSON), прямое чтение/grep по `results/` | +| MCP | запуск `mm_search.py`, `python mm_search.py …`, любой shell вместо MCP tools | + +**Разрешено без спроса:** `mm_search` с индексом (`used_index: true`) — обычный +`search_mode="message"`, `file_name`, `file_content`. + +**Если индекс недоступен или нужен regex/scan:** остановиться, сообщить пользователю +(нет `db_exists`, нужен `mm_index.py`, или требуется медленный обход JSON) и **дождаться +явного «да»** перед `no_index`, `regex`, CLI или чтением файлов экспорта. + +## Prerequisites + +- **`mattermost-search`** MCP server connected (`mm_search`, `mm_search_index_status` visible). +- Export JSON under `results/` (or `MM_SEARCH_ROOT`). +- SQLite index `mattermost_index.sqlite` (or `MM_SEARCH_DB`) for fast search and + **required** for attachment search. Build/update: `python mm_index.py`. + +Server env (set in `mcp.json` `cwd` / `env`): + +| Variable | Default | +|---|---| +| `MM_SEARCH_ROOT` | `results` | +| `MM_SEARCH_DB` | `mattermost_index.sqlite` | + +## Quick start + +1. `mm_search_index_status()` — confirm `db_exists`, `root_exists`, `json_files`. +2. Pick **search_mode** (see table below). +3. `mm_search(query=…, …)` with filters; keep `limit` small (10–50) unless the user needs all hits. +4. Read `matches[].post.message`; for attachments also `matches[].attachment`. + +## Tools + +| Need | Tool | +|---|---| +| Search messages / attachments | `mm_search(query, …)` | +| Check index & export layout | `mm_search_index_status(db?, root?)` | + +## Choose search_mode + +| Goal | `search_mode` | Notes | +|---|---|---| +| Text in post body | `message` (default) | Uses FTS index; JSON scan only with user permission | +| Attachment filename | `file_name` | Index **required**; no `regex` | +| Text inside attachment | `file_content` | Index **required**; UTF-8 text files only | +| Regex in message | `message` + `regex=true` | **User permission required** — scans JSON, skips index | + +## mm_search parameters + +| Param | Role | +|---|---| +| `query` | Search string (or regex pattern) | +| `user` | Exact username filter | +| `channel` | Exact `display_name` or `name` | +| `team` | Exact team name | +| `after` / `before` | `YYYY-MM-DD` or ISO datetime | +| `limit` | Max matches (apply early — large exports) | +| `ignore_case` | Default: on for substring, off for regex | +| `no_index` | Force JSON file scan — **user permission required** | +| `output` | `json` (default) or `human` (CLI-style lines) | +| `show_file` | Prefix `file:idx:` in human output | + +## Response shape (`output=json`) + +```json +{ + "count": 2, + "used_index": true, + "search_mode": "message", + "root": "results", + "db": "mattermost_index.sqlite", + "matches": [ + { + "file": "results/20260605/user/channel.json", + "channel": "general", + "team": "myteam", + "post": { + "id": "…", + "idx": 42, + "created": "2026-01-15T10:00:00Z", + "username": "alice", + "message": "…", + "files": ["path/to/file.pdf"] + }, + "attachment": { + "filename": "doc.txt", + "path": "…", + "is_text": true + } + } + ] +} +``` + +`attachment` is present only for `file_name` / `file_content` hits. + +## Common workflows + +**Find what someone wrote about a topic** + +``` +mm_search(query="deploy rollback", user="alice", after="2026-01-01", limit=20) +``` + +**Find posts in a channel** + +``` +mm_search(query="incident", channel="alerts", limit=30) +``` + +**Find attachment by name** + +``` +mm_search(query="report.pdf", search_mode="file_name", limit=10) +``` + +**Regex over raw JSON (no index)** — только после явного разрешения пользователя + +``` +mm_search(query="ERROR-\\d+", regex=true, no_index=true, limit=50) +``` + +**Human-readable listing for the user** + +``` +mm_search(query="standup", output="human", show_file=true, limit=15) +``` + +## Pitfalls + +- `file_name` / `file_content` + `regex=true` → error. +- Attachment search without index → error; run `mm_index.py` first. +- `channel` / `user` / `team` are **exact** matches, not substring. +- Token budget: prefer `limit`, narrow `after`/`before`, then widen if empty. +- If `count=0` with index: broaden `query`, drop filters, suggest `mm_index.py` — **не** + переключаться на `no_index` / `regex` / CLI без явного разрешения. + +## Related CLI + +Использовать **только по явной просьбе пользователя** (обход MCP): + +| CLI | MCP equivalent | +|---|---| +| `mm_search.py QUERY` | `mm_search(query=QUERY)` | +| `--file-name` | `search_mode="file_name"` | +| `--file-content` | `search_mode="file_content"` | +| `--regex` | `regex=true` (permission required) | +| `--no-index` | `no_index=true` (permission required) | +| `mm_index.py` | not in MCP — предложить пользователю, не запускать самому | diff --git a/mm_search_mcp.py b/mm_search_mcp.py new file mode 100644 index 0000000..be29419 --- /dev/null +++ b/mm_search_mcp.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""MCP server (stdio) exposing mm_search.py search capabilities. + +Cursor / Claude Desktop config example (~/.cursor/mcp.json): + + { + "mcpServers": { + "mattermost-search": { + "command": "/path/to/mattermost-dl/.venv/bin/python", + "args": ["/path/to/mattermost-dl/mm_search_mcp.py"], + "cwd": "/path/to/mattermost-dl", + "env": { + "MM_SEARCH_ROOT": "results", + "MM_SEARCH_DB": "mattermost_index.sqlite" + } + } + } + } + +Requires: pip install fastmcp +""" +from __future__ import annotations + +import datetime as dt +import os +from typing import Any, Literal, Optional + +from fastmcp import FastMCP + +import mm_search as mm_search_lib + +mcp = FastMCP("mattermost-search") + +SearchMode = Literal["message", "file_name", "file_content"] + + +def _env_root() -> str: + return os.environ.get("MM_SEARCH_ROOT", "results") + + +def _env_db() -> str: + return os.environ.get("MM_SEARCH_DB", "mattermost_index.sqlite") + + +def _parse_date(value: str) -> dt.datetime: + try: + return mm_search_lib.parse_iso_date(value) + except Exception as exc: + raise ValueError(f"invalid date '{value}': {exc}") from exc + + +def _collect_matches( + *, + query: str, + root: str, + user: Optional[str], + channel: Optional[str], + team: Optional[str], + after: Optional[dt.datetime], + before: Optional[dt.datetime], + regex: bool, + search_mode: SearchMode, + ignore_case: Optional[bool], + limit: Optional[int], + db: str, + no_index: bool, +) -> list[dict[str, Any]]: + if search_mode != "message" and regex: + raise ValueError("search_mode file_name/file_content do not support regex=True") + + effective_ignore_case = ignore_case if ignore_case is not None else (not regex) + matcher = mm_search_lib.make_matcher(query, regex, effective_ignore_case) + use_index = (not no_index) and os.path.exists(db) + + if search_mode in ("file_name", "file_content"): + if not use_index: + raise ValueError( + "Attachment search (file_name/file_content) requires sqlite index. " + "Create it with mm_index.py and do not set no_index=True." + ) + mode = "name" if search_mode == "file_name" else "content" + matches_iter = mm_search_lib.search_attachments_index( + db_path=db, + query=query, + user=user, + channel=channel, + team=team, + after=after, + before=before, + limit=limit, + mode=mode, + ) + output_limit = None + elif use_index and not regex: + matches_iter = mm_search_lib.search_index( + db_path=db, + query=query, + user=user, + channel=channel, + team=team, + after=after, + before=before, + limit=limit, + ignore_case=effective_ignore_case, + ) + output_limit = None + else: + files = list(mm_search_lib.iter_result_files(root)) + matches_iter = ( + m + for path in files + for m in mm_search_lib.search_file( + path=path, + matcher=matcher, + user=user, + channel=channel, + team=team, + after=after, + before=before, + ) + ) + output_limit = limit + + out: list[dict[str, Any]] = [] + count = 0 + for match in matches_iter: + if output_limit is not None and count >= output_limit: + break + out.append(match) + count += 1 + return out + + +def _format_human(matches: list[dict[str, Any]], show_file: bool) -> str: + lines: list[str] = [] + for m in matches: + p = m["post"] + created = p.get("created", "") + username = p.get("username", "") + msg = (p.get("message") or "").replace("\n", "\\n") + attachment = m.get("attachment") + attachment_suffix = "" + if isinstance(attachment, dict) and attachment.get("filename"): + attachment_suffix = f" [file: {attachment['filename']}]" + if show_file: + file_rel = os.path.relpath(m["file"]) + idx = p.get("idx", "?") + prefix = f"{file_rel}:{idx}: " + else: + prefix = "" + channel = m.get("channel") or "?" + team = m.get("team") or "" + team_prefix = f"[{team}]" if team else "" + lines.append(f"{prefix}{team_prefix}[{channel}] {created} {username}: {msg}{attachment_suffix}") + return "\n".join(lines) + + +@mcp.tool +def mm_search( + query: str, + root: Optional[str] = None, + user: Optional[str] = None, + channel: Optional[str] = None, + team: Optional[str] = None, + after: Optional[str] = None, + before: Optional[str] = None, + regex: bool = False, + search_mode: SearchMode = "message", + ignore_case: Optional[bool] = None, + limit: Optional[int] = None, + db: Optional[str] = None, + no_index: bool = False, + output: Literal["json", "human"] = "json", + show_file: bool = False, +) -> dict[str, Any]: + """Search Mattermost export JSON files produced by mattermost-dl.py. + + Mirrors mm_search.py CLI. Returns structured matches or human-readable lines. + + Args: + query: Search string or regex (when regex=True). + root: Root directory with export results (default: MM_SEARCH_ROOT or "results"). + user: Filter by username (exact match). + channel: Filter by channel display_name or name (exact match). + team: Filter by team name (exact match). + after: Only messages on/after this date (YYYY-MM-DD or ISO). + before: Only messages on/before this date (YYYY-MM-DD or ISO). + regex: Treat query as regular expression (message scan only). + search_mode: "message" (default), "file_name", or "file_content". + ignore_case: Case-insensitive search. Default: True for substring, False for regex. + limit: Maximum number of matches to return. + db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite"). + no_index: Do not use sqlite index even if DB exists. + output: "json" returns match objects; "human" returns CLI-style text lines. + show_file: Include source filename and per-file index in human output. + """ + resolved_root = root or _env_root() + resolved_db = db or _env_db() + parsed_after = _parse_date(after) if after else None + parsed_before = _parse_date(before) if before else None + + matches = _collect_matches( + query=query, + root=resolved_root, + user=user, + channel=channel, + team=team, + after=parsed_after, + before=parsed_before, + regex=regex, + search_mode=search_mode, + ignore_case=ignore_case, + limit=limit, + db=resolved_db, + no_index=no_index, + ) + + result: dict[str, Any] = { + "count": len(matches), + "used_index": (not no_index) + and os.path.exists(resolved_db) + and (search_mode in ("file_name", "file_content") or not regex), + "search_mode": search_mode, + "root": resolved_root, + "db": resolved_db, + } + + if output == "human": + result["text"] = _format_human(matches, show_file=show_file) + else: + result["matches"] = matches + + return result + + +@mcp.tool +def mm_search_index_status( + db: Optional[str] = None, + root: Optional[str] = None, +) -> dict[str, Any]: + """Report whether the sqlite index and export root are available.""" + resolved_db = db or _env_db() + resolved_root = root or _env_root() + db_exists = os.path.exists(resolved_db) + root_exists = os.path.isdir(resolved_root) + + json_count = 0 + if root_exists: + json_count = sum(1 for _ in mm_search_lib.iter_result_files(resolved_root)) + + return { + "db": resolved_db, + "db_exists": db_exists, + "root": resolved_root, + "root_exists": root_exists, + "json_files": json_count, + "index_required_for": ["file_name", "file_content"], + } + + +if __name__ == "__main__": + mcp.run(transport="stdio")