add mcp

mattermost-dl.py

@@ -1,6 +1,8 @@
 import os
 import pickle
+import random
 import sqlite3
+import time
 from datetime import datetime, date, timezone
 from typing import Tuple, Dict, List, Optional
 import getpass
@@ -15,6 +17,80 @@ import json
 GITLAB_URL_DEFAULT = "https://mmgit.eltex-co.ru"
 COOKIE_FILE_DEFAULT = "cookies.pkl"
 
+THROTTLE_DEFAULTS = {
+    "min_delay": 0.4,
+    "max_delay": 1.8,
+    "burst_pause_every": 12,
+    "burst_pause_min": 3.0,
+    "burst_pause_max": 8.0,
+    "file_delay_multiplier": 1.5,
+}
+
+
+class RequestThrottler:
+    """Paces API calls with jitter and occasional longer pauses."""
+
+    def __init__(self, **opts):
+        self.min_delay = opts["min_delay"]
+        self.max_delay = opts["max_delay"]
+        self.burst_pause_every = opts["burst_pause_every"]
+        self.burst_pause_min = opts["burst_pause_min"]
+        self.burst_pause_max = opts["burst_pause_max"]
+        self.file_delay_multiplier = opts["file_delay_multiplier"]
+        self._request_count = 0
+        self._last_request_at = 0.0
+
+    def wait(self, endpoint: str = "") -> None:
+        self._request_count += 1
+        if self.burst_pause_every and self._request_count % self.burst_pause_every == 0:
+            delay = random.uniform(self.burst_pause_min, self.burst_pause_max)
+        else:
+            delay = random.uniform(self.min_delay, self.max_delay)
+
+        if endpoint.startswith("/files/"):
+            delay *= self.file_delay_multiplier
+
+        elapsed = time.monotonic() - self._last_request_at
+        if elapsed < delay:
+            time.sleep(delay - elapsed)
+        self._last_request_at = time.monotonic()
+
+
+def _parse_throttle_config(config: dict) -> Optional[dict]:
+    throttle = config.get("throttle")
+    if not throttle:
+        return None
+    if throttle is True:
+        return dict(THROTTLE_DEFAULTS)
+    if isinstance(throttle, dict):
+        if not throttle.get("enabled", True):
+            return None
+        opts = dict(THROTTLE_DEFAULTS)
+        opts.update({k: v for k, v in throttle.items() if k != "enabled"})
+        return opts
+    return None
+
+
+def apply_throttle(d: Driver, config: dict) -> None:
+    opts = _parse_throttle_config(config)
+    if not opts:
+        return
+
+    throttler = RequestThrottler(**opts)
+    client = d.client
+    original_make_request = client.make_request
+
+    def throttled_make_request(method, endpoint, options=None, params=None, data=None, files=None, basepath=None):
+        throttler.wait(endpoint)
+        return original_make_request(method, endpoint, options, params, data, files, basepath)
+
+    client.make_request = throttled_make_request
+    print(
+        "Request throttling enabled "
+        f"(delay {opts['min_delay']}-{opts['max_delay']}s, "
+        f"pause every {opts['burst_pause_every']} requests)"
+    )
+
 
 def connect(host: str, login_token: str = None, username: str = None, password: str = None) -> Driver:
     d = Driver({
@@ -568,6 +644,7 @@ if __name__ == '__main__':
             config.get("username", None),
             config.get("password", None),
         )
+        apply_throttle(d, config)
     except Exception as e:
         # If token-based authentication failed, drop the token and ask
         # the user to rerun the script so that fresh credentials can be

mattermost-search/SKILL.md

@@ -0,0 +1,172 @@
+---
+name: mattermost-search
+description: |
+  Search Mattermost export dumps via mattermost-search MCP (mm_search_mcp.py):
+  message text, attachment names, attachment text content, filters by user/channel/team/date.
+  Use when the user asks to find messages, posts, or files in Mattermost exports,
+  mattermost-dl results, or the local sqlite index. Requires MCP server mattermost-search.
+  Index bypass (no_index, regex, CLI, grep) needs explicit user permission.
+---
+
+# Mattermost Search (MCP)
+
+Search local Mattermost JSON exports produced by `mattermost-dl.py`. **Always use MCP
+tools** (`mm_search`, `mm_search_index_status`) as the default path.
+
+## Mandatory rules
+
+**Поиск в обход индекса и MCP — только с явного разрешения пользователя.**
+
+Не делать самостоятельно, пока пользователь явно не попросил:
+
+| Обход | Примеры |
+|---|---|
+| Индекс | `no_index=true`, `regex=true` (сканирует JSON), прямое чтение/grep по `results/` |
+| MCP | запуск `mm_search.py`, `python mm_search.py …`, любой shell вместо MCP tools |
+
+**Разрешено без спроса:** `mm_search` с индексом (`used_index: true`) — обычный
+`search_mode="message"`, `file_name`, `file_content`.
+
+**Если индекс недоступен или нужен regex/scan:** остановиться, сообщить пользователю
+(нет `db_exists`, нужен `mm_index.py`, или требуется медленный обход JSON) и **дождаться
+явного «да»** перед `no_index`, `regex`, CLI или чтением файлов экспорта.
+
+## Prerequisites
+
+- **`mattermost-search`** MCP server connected (`mm_search`, `mm_search_index_status` visible).
+- Export JSON under `results/` (or `MM_SEARCH_ROOT`).
+- SQLite index `mattermost_index.sqlite` (or `MM_SEARCH_DB`) for fast search and
+  **required** for attachment search. Build/update: `python mm_index.py`.
+
+Server env (set in `mcp.json` `cwd` / `env`):
+
+| Variable | Default |
+|---|---|
+| `MM_SEARCH_ROOT` | `results` |
+| `MM_SEARCH_DB` | `mattermost_index.sqlite` |
+
+## Quick start
+
+1. `mm_search_index_status()` — confirm `db_exists`, `root_exists`, `json_files`.
+2. Pick **search_mode** (see table below).
+3. `mm_search(query=…, …)` with filters; keep `limit` small (10–50) unless the user needs all hits.
+4. Read `matches[].post.message`; for attachments also `matches[].attachment`.
+
+## Tools
+
+| Need | Tool |
+|---|---|
+| Search messages / attachments | `mm_search(query, …)` |
+| Check index & export layout | `mm_search_index_status(db?, root?)` |
+
+## Choose search_mode
+
+| Goal | `search_mode` | Notes |
+|---|---|---|
+| Text in post body | `message` (default) | Uses FTS index; JSON scan only with user permission |
+| Attachment filename | `file_name` | Index **required**; no `regex` |
+| Text inside attachment | `file_content` | Index **required**; UTF-8 text files only |
+| Regex in message | `message` + `regex=true` | **User permission required** — scans JSON, skips index |
+
+## mm_search parameters
+
+| Param | Role |
+|---|---|
+| `query` | Search string (or regex pattern) |
+| `user` | Exact username filter |
+| `channel` | Exact `display_name` or `name` |
+| `team` | Exact team name |
+| `after` / `before` | `YYYY-MM-DD` or ISO datetime |
+| `limit` | Max matches (apply early — large exports) |
+| `ignore_case` | Default: on for substring, off for regex |
+| `no_index` | Force JSON file scan — **user permission required** |
+| `output` | `json` (default) or `human` (CLI-style lines) |
+| `show_file` | Prefix `file:idx:` in human output |
+
+## Response shape (`output=json`)
+
+```json
+{
+  "count": 2,
+  "used_index": true,
+  "search_mode": "message",
+  "root": "results",
+  "db": "mattermost_index.sqlite",
+  "matches": [
+    {
+      "file": "results/20260605/user/channel.json",
+      "channel": "general",
+      "team": "myteam",
+      "post": {
+        "id": "…",
+        "idx": 42,
+        "created": "2026-01-15T10:00:00Z",
+        "username": "alice",
+        "message": "…",
+        "files": ["path/to/file.pdf"]
+      },
+      "attachment": {
+        "filename": "doc.txt",
+        "path": "…",
+        "is_text": true
+      }
+    }
+  ]
+}
+```
+
+`attachment` is present only for `file_name` / `file_content` hits.
+
+## Common workflows
+
+**Find what someone wrote about a topic**
+
+```
+mm_search(query="deploy rollback", user="alice", after="2026-01-01", limit=20)
+```
+
+**Find posts in a channel**
+
+```
+mm_search(query="incident", channel="alerts", limit=30)
+```
+
+**Find attachment by name**
+
+```
+mm_search(query="report.pdf", search_mode="file_name", limit=10)
+```
+
+**Regex over raw JSON (no index)** — только после явного разрешения пользователя
+
+```
+mm_search(query="ERROR-\\d+", regex=true, no_index=true, limit=50)
+```
+
+**Human-readable listing for the user**
+
+```
+mm_search(query="standup", output="human", show_file=true, limit=15)
+```
+
+## Pitfalls
+
+- `file_name` / `file_content` + `regex=true` → error.
+- Attachment search without index → error; run `mm_index.py` first.
+- `channel` / `user` / `team` are **exact** matches, not substring.
+- Token budget: prefer `limit`, narrow `after`/`before`, then widen if empty.
+- If `count=0` with index: broaden `query`, drop filters, suggest `mm_index.py` — **не**
+  переключаться на `no_index` / `regex` / CLI без явного разрешения.
+
+## Related CLI
+
+Использовать **только по явной просьбе пользователя** (обход MCP):
+
+| CLI | MCP equivalent |
+|---|---|
+| `mm_search.py QUERY` | `mm_search(query=QUERY)` |
+| `--file-name` | `search_mode="file_name"` |
+| `--file-content` | `search_mode="file_content"` |
+| `--regex` | `regex=true` (permission required) |
+| `--no-index` | `no_index=true` (permission required) |
+| `mm_index.py` | not in MCP — предложить пользователю, не запускать самому |

mm_search_mcp.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+"""MCP server (stdio) exposing mm_search.py search capabilities.
+
+Cursor / Claude Desktop config example (~/.cursor/mcp.json):
+
+    {
+      "mcpServers": {
+        "mattermost-search": {
+          "command": "/path/to/mattermost-dl/.venv/bin/python",
+          "args": ["/path/to/mattermost-dl/mm_search_mcp.py"],
+          "cwd": "/path/to/mattermost-dl",
+          "env": {
+            "MM_SEARCH_ROOT": "results",
+            "MM_SEARCH_DB": "mattermost_index.sqlite"
+          }
+        }
+      }
+    }
+
+Requires: pip install fastmcp
+"""
+from __future__ import annotations
+
+import datetime as dt
+import os
+from typing import Any, Literal, Optional
+
+from fastmcp import FastMCP
+
+import mm_search as mm_search_lib
+
+mcp = FastMCP("mattermost-search")
+
+SearchMode = Literal["message", "file_name", "file_content"]
+
+
+def _env_root() -> str:
+    return os.environ.get("MM_SEARCH_ROOT", "results")
+
+
+def _env_db() -> str:
+    return os.environ.get("MM_SEARCH_DB", "mattermost_index.sqlite")
+
+
+def _parse_date(value: str) -> dt.datetime:
+    try:
+        return mm_search_lib.parse_iso_date(value)
+    except Exception as exc:
+        raise ValueError(f"invalid date '{value}': {exc}") from exc
+
+
+def _collect_matches(
+    *,
+    query: str,
+    root: str,
+    user: Optional[str],
+    channel: Optional[str],
+    team: Optional[str],
+    after: Optional[dt.datetime],
+    before: Optional[dt.datetime],
+    regex: bool,
+    search_mode: SearchMode,
+    ignore_case: Optional[bool],
+    limit: Optional[int],
+    db: str,
+    no_index: bool,
+) -> list[dict[str, Any]]:
+    if search_mode != "message" and regex:
+        raise ValueError("search_mode file_name/file_content do not support regex=True")
+
+    effective_ignore_case = ignore_case if ignore_case is not None else (not regex)
+    matcher = mm_search_lib.make_matcher(query, regex, effective_ignore_case)
+    use_index = (not no_index) and os.path.exists(db)
+
+    if search_mode in ("file_name", "file_content"):
+        if not use_index:
+            raise ValueError(
+                "Attachment search (file_name/file_content) requires sqlite index. "
+                "Create it with mm_index.py and do not set no_index=True."
+            )
+        mode = "name" if search_mode == "file_name" else "content"
+        matches_iter = mm_search_lib.search_attachments_index(
+            db_path=db,
+            query=query,
+            user=user,
+            channel=channel,
+            team=team,
+            after=after,
+            before=before,
+            limit=limit,
+            mode=mode,
+        )
+        output_limit = None
+    elif use_index and not regex:
+        matches_iter = mm_search_lib.search_index(
+            db_path=db,
+            query=query,
+            user=user,
+            channel=channel,
+            team=team,
+            after=after,
+            before=before,
+            limit=limit,
+            ignore_case=effective_ignore_case,
+        )
+        output_limit = None
+    else:
+        files = list(mm_search_lib.iter_result_files(root))
+        matches_iter = (
+            m
+            for path in files
+            for m in mm_search_lib.search_file(
+                path=path,
+                matcher=matcher,
+                user=user,
+                channel=channel,
+                team=team,
+                after=after,
+                before=before,
+            )
+        )
+        output_limit = limit
+
+    out: list[dict[str, Any]] = []
+    count = 0
+    for match in matches_iter:
+        if output_limit is not None and count >= output_limit:
+            break
+        out.append(match)
+        count += 1
+    return out
+
+
+def _format_human(matches: list[dict[str, Any]], show_file: bool) -> str:
+    lines: list[str] = []
+    for m in matches:
+        p = m["post"]
+        created = p.get("created", "")
+        username = p.get("username", "")
+        msg = (p.get("message") or "").replace("\n", "\\n")
+        attachment = m.get("attachment")
+        attachment_suffix = ""
+        if isinstance(attachment, dict) and attachment.get("filename"):
+            attachment_suffix = f" [file: {attachment['filename']}]"
+        if show_file:
+            file_rel = os.path.relpath(m["file"])
+            idx = p.get("idx", "?")
+            prefix = f"{file_rel}:{idx}: "
+        else:
+            prefix = ""
+        channel = m.get("channel") or "?"
+        team = m.get("team") or ""
+        team_prefix = f"[{team}]" if team else ""
+        lines.append(f"{prefix}{team_prefix}[{channel}] {created} {username}: {msg}{attachment_suffix}")
+    return "\n".join(lines)
+
+
+@mcp.tool
+def mm_search(
+    query: str,
+    root: Optional[str] = None,
+    user: Optional[str] = None,
+    channel: Optional[str] = None,
+    team: Optional[str] = None,
+    after: Optional[str] = None,
+    before: Optional[str] = None,
+    regex: bool = False,
+    search_mode: SearchMode = "message",
+    ignore_case: Optional[bool] = None,
+    limit: Optional[int] = None,
+    db: Optional[str] = None,
+    no_index: bool = False,
+    output: Literal["json", "human"] = "json",
+    show_file: bool = False,
+) -> dict[str, Any]:
+    """Search Mattermost export JSON files produced by mattermost-dl.py.
+
+    Mirrors mm_search.py CLI. Returns structured matches or human-readable lines.
+
+    Args:
+        query: Search string or regex (when regex=True).
+        root: Root directory with export results (default: MM_SEARCH_ROOT or "results").
+        user: Filter by username (exact match).
+        channel: Filter by channel display_name or name (exact match).
+        team: Filter by team name (exact match).
+        after: Only messages on/after this date (YYYY-MM-DD or ISO).
+        before: Only messages on/before this date (YYYY-MM-DD or ISO).
+        regex: Treat query as regular expression (message scan only).
+        search_mode: "message" (default), "file_name", or "file_content".
+        ignore_case: Case-insensitive search. Default: True for substring, False for regex.
+        limit: Maximum number of matches to return.
+        db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite").
+        no_index: Do not use sqlite index even if DB exists.
+        output: "json" returns match objects; "human" returns CLI-style text lines.
+        show_file: Include source filename and per-file index in human output.
+    """
+    resolved_root = root or _env_root()
+    resolved_db = db or _env_db()
+    parsed_after = _parse_date(after) if after else None
+    parsed_before = _parse_date(before) if before else None
+
+    matches = _collect_matches(
+        query=query,
+        root=resolved_root,
+        user=user,
+        channel=channel,
+        team=team,
+        after=parsed_after,
+        before=parsed_before,
+        regex=regex,
+        search_mode=search_mode,
+        ignore_case=ignore_case,
+        limit=limit,
+        db=resolved_db,
+        no_index=no_index,
+    )
+
+    result: dict[str, Any] = {
+        "count": len(matches),
+        "used_index": (not no_index)
+        and os.path.exists(resolved_db)
+        and (search_mode in ("file_name", "file_content") or not regex),
+        "search_mode": search_mode,
+        "root": resolved_root,
+        "db": resolved_db,
+    }
+
+    if output == "human":
+        result["text"] = _format_human(matches, show_file=show_file)
+    else:
+        result["matches"] = matches
+
+    return result
+
+
+@mcp.tool
+def mm_search_index_status(
+    db: Optional[str] = None,
+    root: Optional[str] = None,
+) -> dict[str, Any]:
+    """Report whether the sqlite index and export root are available."""
+    resolved_db = db or _env_db()
+    resolved_root = root or _env_root()
+    db_exists = os.path.exists(resolved_db)
+    root_exists = os.path.isdir(resolved_root)
+
+    json_count = 0
+    if root_exists:
+        json_count = sum(1 for _ in mm_search_lib.iter_result_files(resolved_root))
+
+    return {
+        "db": resolved_db,
+        "db_exists": db_exists,
+        "root": resolved_root,
+        "root_exists": root_exists,
+        "json_files": json_count,
+        "index_required_for": ["file_name", "file_content"],
+    }
+
+
+if __name__ == "__main__":
+    mcp.run(transport="stdio")