grayhook/mattermost-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add mcp

Browse Source
This commit is contained in:
Сергей Маринкевич
2026-06-05 15:09:07 +07:00
parent 808de8562f
commit 99dc9f173d
3 changed files with 511 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mattermost-dl.py
+77
View File
@@ -1,6 +1,8 @@
import os
import pickle
import random
import sqlite3
import time
from datetime import datetime, date, timezone
from typing import Tuple, Dict, List, Optional
import getpass
@@ -15,6 +17,80 @@ import json
GITLAB_URL_DEFAULT = "https://mmgit.eltex-co.ru"
COOKIE_FILE_DEFAULT = "cookies.pkl"
THROTTLE_DEFAULTS = {
"min_delay": 0.4,
"max_delay": 1.8,
"burst_pause_every": 12,
"burst_pause_min": 3.0,
"burst_pause_max": 8.0,
"file_delay_multiplier": 1.5,
}
class RequestThrottler:
"""Paces API calls with jitter and occasional longer pauses."""
def __init__(self, **opts):
self.min_delay = opts["min_delay"]
self.max_delay = opts["max_delay"]
self.burst_pause_every = opts["burst_pause_every"]
self.burst_pause_min = opts["burst_pause_min"]
self.burst_pause_max = opts["burst_pause_max"]
self.file_delay_multiplier = opts["file_delay_multiplier"]
self._request_count = 0
self._last_request_at = 0.0
def wait(self, endpoint: str = "") -> None:
self._request_count += 1
if self.burst_pause_every and self._request_count % self.burst_pause_every == 0:
delay = random.uniform(self.burst_pause_min, self.burst_pause_max)
else:
delay = random.uniform(self.min_delay, self.max_delay)
if endpoint.startswith("/files/"):
delay *= self.file_delay_multiplier
elapsed = time.monotonic() - self._last_request_at
if elapsed < delay:
time.sleep(delay - elapsed)
self._last_request_at = time.monotonic()
def _parse_throttle_config(config: dict) -> Optional[dict]:
throttle = config.get("throttle")
if not throttle:
return None
if throttle is True:
return dict(THROTTLE_DEFAULTS)
if isinstance(throttle, dict):
if not throttle.get("enabled", True):
return None
opts = dict(THROTTLE_DEFAULTS)
opts.update({k: v for k, v in throttle.items() if k != "enabled"})
return opts
return None
def apply_throttle(d: Driver, config: dict) -> None:
opts = _parse_throttle_config(config)
if not opts:
return
throttler = RequestThrottler(**opts)
client = d.client
original_make_request = client.make_request
def throttled_make_request(method, endpoint, options=None, params=None, data=None, files=None, basepath=None):
throttler.wait(endpoint)
return original_make_request(method, endpoint, options, params, data, files, basepath)
client.make_request = throttled_make_request
print(
"Request throttling enabled "
f"(delay {opts['min_delay']}-{opts['max_delay']}s, "
f"pause every {opts['burst_pause_every']} requests)"
)
def connect(host: str, login_token: str = None, username: str = None, password: str = None) -> Driver:
d = Driver({
@@ -568,6 +644,7 @@ if __name__ == '__main__':
config.get("username", None),
config.get("password", None),
)
apply_throttle(d, config)
except Exception as e:
# If token-based authentication failed, drop the token and ask
# the user to rerun the script so that fresh credentials can be
mattermost-search/SKILL.md
+172
View File
@@ -0,0 +1,172 @@
---
name: mattermost-search
description: |
Search Mattermost export dumps via mattermost-search MCP (mm_search_mcp.py):
message text, attachment names, attachment text content, filters by user/channel/team/date.
Use when the user asks to find messages, posts, or files in Mattermost exports,
mattermost-dl results, or the local sqlite index. Requires MCP server mattermost-search.
Index bypass (no_index, regex, CLI, grep) needs explicit user permission.
---
# Mattermost Search (MCP)
Search local Mattermost JSON exports produced by `mattermost-dl.py`. **Always use MCP
tools** (`mm_search`, `mm_search_index_status`) as the default path.
## Mandatory rules
**Поиск в обход индекса и MCP — только с явного разрешения пользователя.**
Не делать самостоятельно, пока пользователь явно не попросил:
| Обход | Примеры |
|---|---|
| Индекс | `no_index=true`, `regex=true` (сканирует JSON), прямое чтение/grep по `results/` |
| MCP | запуск `mm_search.py`, `python mm_search.py …`, любой shell вместо MCP tools |
**Разрешено без спроса:** `mm_search` с индексом (`used_index: true`) — обычный
`search_mode="message"`, `file_name`, `file_content`.
**Если индекс недоступен или нужен regex/scan:** остановиться, сообщить пользователю
(нет `db_exists`, нужен `mm_index.py`, или требуется медленный обход JSON) и **дождаться
явного «да»** перед `no_index`, `regex`, CLI или чтением файлов экспорта.
## Prerequisites
- **`mattermost-search`** MCP server connected (`mm_search`, `mm_search_index_status` visible).
- Export JSON under `results/` (or `MM_SEARCH_ROOT`).
- SQLite index `mattermost_index.sqlite` (or `MM_SEARCH_DB`) for fast search and
**required** for attachment search. Build/update: `python mm_index.py`.
Server env (set in `mcp.json` `cwd` / `env`):
| Variable | Default |
|---|---|
| `MM_SEARCH_ROOT` | `results` |
| `MM_SEARCH_DB` | `mattermost_index.sqlite` |
## Quick start
1. `mm_search_index_status()` — confirm `db_exists`, `root_exists`, `json_files`.
2. Pick **search_mode** (see table below).
3. `mm_search(query=…, …)` with filters; keep `limit` small (10–50) unless the user needs all hits.
4. Read `matches[].post.message`; for attachments also `matches[].attachment`.
## Tools
| Need | Tool |
|---|---|
| Search messages / attachments | `mm_search(query, …)` |
| Check index & export layout | `mm_search_index_status(db?, root?)` |
## Choose search_mode
| Goal | `search_mode` | Notes |
|---|---|---|
| Text in post body | `message` (default) | Uses FTS index; JSON scan only with user permission |
| Attachment filename | `file_name` | Index **required**; no `regex` |
| Text inside attachment | `file_content` | Index **required**; UTF-8 text files only |
| Regex in message | `message` + `regex=true` | **User permission required** — scans JSON, skips index |
## mm_search parameters
| Param | Role |
|---|---|
| `query` | Search string (or regex pattern) |
| `user` | Exact username filter |
| `channel` | Exact `display_name` or `name` |
| `team` | Exact team name |
| `after` / `before` | `YYYY-MM-DD` or ISO datetime |
| `limit` | Max matches (apply early — large exports) |
| `ignore_case` | Default: on for substring, off for regex |
| `no_index` | Force JSON file scan — **user permission required** |
| `output` | `json` (default) or `human` (CLI-style lines) |
| `show_file` | Prefix `file:idx:` in human output |
## Response shape (`output=json`)
```json
{
"count": 2,
"used_index": true,
"search_mode": "message",
"root": "results",
"db": "mattermost_index.sqlite",
"matches": [
{
"file": "results/20260605/user/channel.json",
"channel": "general",
"team": "myteam",
"post": {
"id": "…",
"idx": 42,
"created": "2026-01-15T10:00:00Z",
"username": "alice",
"message": "…",
"files": ["path/to/file.pdf"]
},
"attachment": {
"filename": "doc.txt",
"path": "…",
"is_text": true
}
}
]
}
```
`attachment` is present only for `file_name` / `file_content` hits.
## Common workflows
**Find what someone wrote about a topic**
```
mm_search(query="deploy rollback", user="alice", after="2026-01-01", limit=20)
```
**Find posts in a channel**
```
mm_search(query="incident", channel="alerts", limit=30)
```
**Find attachment by name**
```
mm_search(query="report.pdf", search_mode="file_name", limit=10)
```
**Regex over raw JSON (no index)** — только после явного разрешения пользователя
```
mm_search(query="ERROR-\\d+", regex=true, no_index=true, limit=50)
```
**Human-readable listing for the user**
```
mm_search(query="standup", output="human", show_file=true, limit=15)
```
## Pitfalls
- `file_name` / `file_content` + `regex=true` → error.
- Attachment search without index → error; run `mm_index.py` first.
- `channel` / `user` / `team` are **exact** matches, not substring.
- Token budget: prefer `limit`, narrow `after`/`before`, then widen if empty.
- If `count=0` with index: broaden `query`, drop filters, suggest `mm_index.py` — **не**
переключаться на `no_index` / `regex` / CLI без явного разрешения.
## Related CLI
Использовать **только по явной просьбе пользователя** (обход MCP):
| CLI | MCP equivalent |
|---|---|
| `mm_search.py QUERY` | `mm_search(query=QUERY)` |
| `--file-name` | `search_mode="file_name"` |
| `--file-content` | `search_mode="file_content"` |
| `--regex` | `regex=true` (permission required) |
| `--no-index` | `no_index=true` (permission required) |
| `mm_index.py` | not in MCP — предложить пользователю, не запускать самому |
mm_search_mcp.py
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env python3
"""MCP server (stdio) exposing mm_search.py search capabilities.
Cursor / Claude Desktop config example (~/.cursor/mcp.json):
{
"mcpServers": {
"mattermost-search": {
"command": "/path/to/mattermost-dl/.venv/bin/python",
"args": ["/path/to/mattermost-dl/mm_search_mcp.py"],
"cwd": "/path/to/mattermost-dl",
"env": {
"MM_SEARCH_ROOT": "results",
"MM_SEARCH_DB": "mattermost_index.sqlite"
}
}
}
}
Requires: pip install fastmcp
"""
from __future__ import annotations
import datetime as dt
import os
from typing import Any, Literal, Optional
from fastmcp import FastMCP
import mm_search as mm_search_lib
mcp = FastMCP("mattermost-search")
SearchMode = Literal["message", "file_name", "file_content"]
def _env_root() -> str:
return os.environ.get("MM_SEARCH_ROOT", "results")
def _env_db() -> str:
return os.environ.get("MM_SEARCH_DB", "mattermost_index.sqlite")
def _parse_date(value: str) -> dt.datetime:
try:
return mm_search_lib.parse_iso_date(value)
except Exception as exc:
raise ValueError(f"invalid date '{value}': {exc}") from exc
def _collect_matches(
*,
query: str,
root: str,
user: Optional[str],
channel: Optional[str],
team: Optional[str],
after: Optional[dt.datetime],
before: Optional[dt.datetime],
regex: bool,
search_mode: SearchMode,
ignore_case: Optional[bool],
limit: Optional[int],
db: str,
no_index: bool,
) -> list[dict[str, Any]]:
if search_mode != "message" and regex:
raise ValueError("search_mode file_name/file_content do not support regex=True")
effective_ignore_case = ignore_case if ignore_case is not None else (not regex)
matcher = mm_search_lib.make_matcher(query, regex, effective_ignore_case)
use_index = (not no_index) and os.path.exists(db)
if search_mode in ("file_name", "file_content"):
if not use_index:
raise ValueError(
"Attachment search (file_name/file_content) requires sqlite index. "
"Create it with mm_index.py and do not set no_index=True."
)
mode = "name" if search_mode == "file_name" else "content"
matches_iter = mm_search_lib.search_attachments_index(
db_path=db,
query=query,
user=user,
channel=channel,
team=team,
after=after,
before=before,
limit=limit,
mode=mode,
)
output_limit = None
elif use_index and not regex:
matches_iter = mm_search_lib.search_index(
db_path=db,
query=query,
user=user,
channel=channel,
team=team,
after=after,
before=before,
limit=limit,
ignore_case=effective_ignore_case,
)
output_limit = None
else:
files = list(mm_search_lib.iter_result_files(root))
matches_iter = (
m
for path in files
for m in mm_search_lib.search_file(
path=path,
matcher=matcher,
user=user,
channel=channel,
team=team,
after=after,
before=before,
)
)
output_limit = limit
out: list[dict[str, Any]] = []
count = 0
for match in matches_iter:
if output_limit is not None and count >= output_limit:
break
out.append(match)
count += 1
return out
def _format_human(matches: list[dict[str, Any]], show_file: bool) -> str:
lines: list[str] = []
for m in matches:
p = m["post"]
created = p.get("created", "")
username = p.get("username", "")
msg = (p.get("message") or "").replace("\n", "\\n")
attachment = m.get("attachment")
attachment_suffix = ""
if isinstance(attachment, dict) and attachment.get("filename"):
attachment_suffix = f" [file: {attachment['filename']}]"
if show_file:
file_rel = os.path.relpath(m["file"])
idx = p.get("idx", "?")
prefix = f"{file_rel}:{idx}: "
else:
prefix = ""
channel = m.get("channel") or "?"
team = m.get("team") or ""
team_prefix = f"[{team}]" if team else ""
lines.append(f"{prefix}{team_prefix}[{channel}] {created} {username}: {msg}{attachment_suffix}")
return "\n".join(lines)
@mcp.tool
def mm_search(
query: str,
root: Optional[str] = None,
user: Optional[str] = None,
channel: Optional[str] = None,
team: Optional[str] = None,
after: Optional[str] = None,
before: Optional[str] = None,
regex: bool = False,
search_mode: SearchMode = "message",
ignore_case: Optional[bool] = None,
limit: Optional[int] = None,
db: Optional[str] = None,
no_index: bool = False,
output: Literal["json", "human"] = "json",
show_file: bool = False,
) -> dict[str, Any]:
"""Search Mattermost export JSON files produced by mattermost-dl.py.
Mirrors mm_search.py CLI. Returns structured matches or human-readable lines.
Args:
query: Search string or regex (when regex=True).
root: Root directory with export results (default: MM_SEARCH_ROOT or "results").
user: Filter by username (exact match).
channel: Filter by channel display_name or name (exact match).
team: Filter by team name (exact match).
after: Only messages on/after this date (YYYY-MM-DD or ISO).
before: Only messages on/before this date (YYYY-MM-DD or ISO).
regex: Treat query as regular expression (message scan only).
search_mode: "message" (default), "file_name", or "file_content".
ignore_case: Case-insensitive search. Default: True for substring, False for regex.
limit: Maximum number of matches to return.
db: SQLite index path (default: MM_SEARCH_DB or "mattermost_index.sqlite").
no_index: Do not use sqlite index even if DB exists.
output: "json" returns match objects; "human" returns CLI-style text lines.
show_file: Include source filename and per-file index in human output.
"""
resolved_root = root or _env_root()
resolved_db = db or _env_db()
parsed_after = _parse_date(after) if after else None
parsed_before = _parse_date(before) if before else None
matches = _collect_matches(
query=query,
root=resolved_root,
user=user,
channel=channel,
team=team,
after=parsed_after,
before=parsed_before,
regex=regex,
search_mode=search_mode,
ignore_case=ignore_case,
limit=limit,
db=resolved_db,
no_index=no_index,
)
result: dict[str, Any] = {
"count": len(matches),
"used_index": (not no_index)
and os.path.exists(resolved_db)
and (search_mode in ("file_name", "file_content") or not regex),
"search_mode": search_mode,
"root": resolved_root,
"db": resolved_db,
}
if output == "human":
result["text"] = _format_human(matches, show_file=show_file)
else:
result["matches"] = matches
return result
@mcp.tool
def mm_search_index_status(
db: Optional[str] = None,
root: Optional[str] = None,
) -> dict[str, Any]:
"""Report whether the sqlite index and export root are available."""
resolved_db = db or _env_db()
resolved_root = root or _env_root()
db_exists = os.path.exists(resolved_db)
root_exists = os.path.isdir(resolved_root)
json_count = 0
if root_exists:
json_count = sum(1 for _ in mm_search_lib.iter_result_files(resolved_root))
return {
"db": resolved_db,
"db_exists": db_exists,
"root": resolved_root,
"root_exists": root_exists,
"json_files": json_count,
"index_required_for": ["file_name", "file_content"],
}
if __name__ == "__main__":
mcp.run(transport="stdio")