Add dev stacks
This commit is contained in:
988
stacks/mllogwatcher/scripts/grafana_alert_webhook.py
Executable file
988
stacks/mllogwatcher/scripts/grafana_alert_webhook.py
Executable file
@@ -0,0 +1,988 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal FastAPI web server that accepts Grafana alert webhooks, looks up the
|
||||
matching runbook entry, builds an LLM prompt, and calls OpenRouter to return a
|
||||
triage summary.
|
||||
|
||||
Run with:
|
||||
uvicorn scripts.grafana_alert_webhook:app --host 0.0.0.0 --port 8081
|
||||
|
||||
Environment variables:
|
||||
RUNBOOK_PATH Path to alert_runbook.yaml (default: ./alert_runbook.yaml)
|
||||
OPENROUTER_API_KEY Required; API token for https://openrouter.ai
|
||||
OPENROUTER_MODEL Optional; default openai/gpt-4o-mini
|
||||
OPENROUTER_REFERER Optional referer header
|
||||
OPENROUTER_TITLE Optional title header (default: Grafana Alert Webhook)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
from textwrap import indent
|
||||
import smtplib
|
||||
from email.message import EmailMessage
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from langchain.llms.base import LLM
|
||||
|
||||
LOGGER = logging.getLogger("grafana_webhook")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
RUNBOOK_PATH = Path(os.environ.get("RUNBOOK_PATH", "alert_runbook.yaml"))
|
||||
ANSIBLE_HOSTS_PATH = Path(os.environ.get("ANSIBLE_HOSTS_PATH", "/etc/ansible/hosts"))
|
||||
OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY")
|
||||
OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "openai/gpt-4o-mini")
|
||||
OPENROUTER_REFERER = os.environ.get("OPENROUTER_REFERER")
|
||||
OPENROUTER_TITLE = os.environ.get("OPENROUTER_TITLE", "Grafana Alert Webhook")
|
||||
|
||||
TRIAGE_ENABLE_COMMANDS = os.environ.get("TRIAGE_ENABLE_COMMANDS", "0").lower() in {"1", "true", "yes", "on"}
|
||||
TRIAGE_COMMAND_RUNNER = os.environ.get("TRIAGE_COMMAND_RUNNER", "ssh").lower()
|
||||
TRIAGE_SSH_USER = os.environ.get("TRIAGE_SSH_USER", "root")
|
||||
TRIAGE_SSH_OPTIONS = shlex.split(
|
||||
os.environ.get("TRIAGE_SSH_OPTIONS", "-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5")
|
||||
)
|
||||
TRIAGE_COMMAND_TIMEOUT = int(os.environ.get("TRIAGE_COMMAND_TIMEOUT", "60"))
|
||||
TRIAGE_DEFAULT_OS = os.environ.get("TRIAGE_DEFAULT_OS", "linux").lower()
|
||||
TRIAGE_MAX_COMMANDS = int(os.environ.get("TRIAGE_MAX_COMMANDS", "3"))
|
||||
TRIAGE_OUTPUT_LIMIT = int(os.environ.get("TRIAGE_OUTPUT_LIMIT", "1200"))
|
||||
# LangChain-driven investigation loop
|
||||
TRIAGE_MAX_ITERATIONS = int(os.environ.get("TRIAGE_MAX_ITERATIONS", "3"))
|
||||
TRIAGE_FOLLOWUP_MAX_COMMANDS = int(os.environ.get("TRIAGE_FOLLOWUP_MAX_COMMANDS", "4"))
|
||||
TRIAGE_SYSTEM_PROMPT = os.environ.get(
|
||||
"TRIAGE_SYSTEM_PROMPT",
|
||||
(
|
||||
"You are assisting with on-call investigations. Always reply with JSON containing:\n"
|
||||
"analysis: your findings and next steps.\n"
|
||||
"followup_commands: list of command specs (summary, command, optional runner/os) to gather more data.\n"
|
||||
"complete: true when sufficient information is gathered.\n"
|
||||
"Request commands only when more evidence is required."
|
||||
),
|
||||
)
|
||||
TRIAGE_VERBOSE_LOGS = os.environ.get("TRIAGE_VERBOSE_LOGS", "0").lower() in {"1", "true", "yes", "on"}
|
||||
TRIAGE_EMAIL_ENABLED = os.environ.get("TRIAGE_EMAIL_ENABLED", "0").lower() in {"1", "true", "yes", "on"}
|
||||
TRIAGE_EMAIL_FROM = os.environ.get("TRIAGE_EMAIL_FROM")
|
||||
TRIAGE_EMAIL_TO = [addr.strip() for addr in os.environ.get("TRIAGE_EMAIL_TO", "").split(",") if addr.strip()]
|
||||
TRIAGE_SMTP_HOST = os.environ.get("TRIAGE_SMTP_HOST")
|
||||
TRIAGE_SMTP_PORT = int(os.environ.get("TRIAGE_SMTP_PORT", "587"))
|
||||
TRIAGE_SMTP_USER = os.environ.get("TRIAGE_SMTP_USER")
|
||||
TRIAGE_SMTP_PASSWORD = os.environ.get("TRIAGE_SMTP_PASSWORD")
|
||||
TRIAGE_SMTP_STARTTLS = os.environ.get("TRIAGE_SMTP_STARTTLS", "1").lower() in {"1", "true", "yes", "on"}
|
||||
TRIAGE_SMTP_SSL = os.environ.get("TRIAGE_SMTP_SSL", "0").lower() in {"1", "true", "yes", "on"}
|
||||
TRIAGE_SMTP_TIMEOUT = int(os.environ.get("TRIAGE_SMTP_TIMEOUT", "20"))
|
||||
|
||||
|
||||
def log_verbose(title: str, content: Any) -> None:
|
||||
"""Emit structured verbose logs when TRIAGE_VERBOSE_LOGS is enabled."""
|
||||
if not TRIAGE_VERBOSE_LOGS:
|
||||
return
|
||||
if isinstance(content, (dict, list)):
|
||||
text = json.dumps(content, indent=2, sort_keys=True)
|
||||
else:
|
||||
text = str(content)
|
||||
LOGGER.info("%s:\n%s", title, text)
|
||||
|
||||
|
||||
def email_notifications_configured() -> bool:
|
||||
if not TRIAGE_EMAIL_ENABLED:
|
||||
return False
|
||||
if not (TRIAGE_SMTP_HOST and TRIAGE_EMAIL_FROM and TRIAGE_EMAIL_TO):
|
||||
LOGGER.warning(
|
||||
"Email notifications requested but TRIAGE_SMTP_HOST/TRIAGE_EMAIL_FROM/TRIAGE_EMAIL_TO are incomplete."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def format_command_results_for_email(results: List[Dict[str, Any]]) -> str:
|
||||
if not results:
|
||||
return "No automation commands were executed."
|
||||
lines: List[str] = []
|
||||
for result in results:
|
||||
lines.append(f"- {result.get('summary')} [{result.get('status')}] {result.get('command')}")
|
||||
stdout = result.get("stdout")
|
||||
stderr = result.get("stderr")
|
||||
error = result.get("error")
|
||||
if stdout:
|
||||
lines.append(indent(truncate_text(stdout, 800), " stdout: "))
|
||||
if stderr:
|
||||
lines.append(indent(truncate_text(stderr, 800), " stderr: "))
|
||||
if error and result.get("status") != "ok":
|
||||
lines.append(f" error: {error}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_email_body(alert: Dict[str, Any], result: Dict[str, Any], context: Dict[str, Any]) -> str:
|
||||
lines = [
|
||||
f"Alert: {result.get('alertname')} ({result.get('rule_uid')})",
|
||||
f"Host: {result.get('host') or context.get('host')}",
|
||||
f"Status: {alert.get('status')}",
|
||||
f"Value: {alert.get('value') or alert.get('annotations', {}).get('value')}",
|
||||
f"Grafana Rule: {context.get('rule_url')}",
|
||||
"",
|
||||
"LLM Summary:",
|
||||
result.get("llm_summary") or "(no summary returned)",
|
||||
"",
|
||||
"Command Results:",
|
||||
format_command_results_for_email(result.get("command_results") or []),
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def send_summary_email(alert: Dict[str, Any], result: Dict[str, Any], context: Dict[str, Any]) -> None:
|
||||
if not email_notifications_configured():
|
||||
return
|
||||
subject_host = result.get("host") or context.get("host") or "(unknown host)"
|
||||
subject = f"[Grafana] {result.get('alertname')} - {subject_host}"
|
||||
body = build_email_body(alert, result, context)
|
||||
message = EmailMessage()
|
||||
message["Subject"] = subject
|
||||
message["From"] = TRIAGE_EMAIL_FROM
|
||||
message["To"] = ", ".join(TRIAGE_EMAIL_TO)
|
||||
message.set_content(body)
|
||||
try:
|
||||
smtp_class = smtplib.SMTP_SSL if TRIAGE_SMTP_SSL else smtplib.SMTP
|
||||
with smtp_class(TRIAGE_SMTP_HOST, TRIAGE_SMTP_PORT, timeout=TRIAGE_SMTP_TIMEOUT) as client:
|
||||
if TRIAGE_SMTP_STARTTLS and not TRIAGE_SMTP_SSL:
|
||||
client.starttls()
|
||||
if TRIAGE_SMTP_USER:
|
||||
client.login(TRIAGE_SMTP_USER, TRIAGE_SMTP_PASSWORD or "")
|
||||
client.send_message(message)
|
||||
LOGGER.info("Sent summary email to %s for host %s", ", ".join(TRIAGE_EMAIL_TO), subject_host)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
LOGGER.exception("Failed to send summary email: %s", exc)
|
||||
|
||||
app = FastAPI(title="Grafana Alert Webhook", version="1.0.0")
|
||||
|
||||
_RUNBOOK_INDEX: Dict[str, Dict[str, Any]] = {}
|
||||
_INVENTORY_INDEX: Dict[str, Dict[str, Any]] = {}
|
||||
_INVENTORY_GROUP_VARS: Dict[str, Dict[str, str]] = {}
|
||||
_TEMPLATE_PATTERN = re.compile(r"{{\s*([a-zA-Z0-9_]+)\s*}}")
|
||||
|
||||
|
||||
DEFAULT_SYSTEM_PROMPT = TRIAGE_SYSTEM_PROMPT
|
||||
|
||||
|
||||
class OpenRouterLLM(LLM):
|
||||
"""LangChain-compatible LLM that calls OpenRouter chat completions."""
|
||||
|
||||
api_key: str
|
||||
model_name: str
|
||||
|
||||
def __init__(self, api_key: str, model_name: str, **kwargs: Any) -> None:
|
||||
super().__init__(api_key=api_key, model_name=model_name, **kwargs)
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "openrouter"
|
||||
|
||||
def __call__(self, prompt: str, stop: Optional[List[str]] = None) -> str:
|
||||
return self._call(prompt, stop=stop)
|
||||
|
||||
def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
|
||||
payload = {
|
||||
"model": self.model_name,
|
||||
"messages": [
|
||||
{"role": "system", "content": DEFAULT_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
}
|
||||
log_verbose("OpenRouter request payload", payload)
|
||||
if stop:
|
||||
payload["stop"] = stop
|
||||
LOGGER.info("Posting to OpenRouter model=%s via LangChain", self.model_name)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if OPENROUTER_REFERER:
|
||||
headers["HTTP-Referer"] = OPENROUTER_REFERER
|
||||
if OPENROUTER_TITLE:
|
||||
headers["X-Title"] = OPENROUTER_TITLE
|
||||
response = requests.post("https://openrouter.ai/api/v1/chat/completions", json=payload, headers=headers, timeout=90)
|
||||
if response.status_code >= 400:
|
||||
try:
|
||||
detail = response.json()
|
||||
except ValueError:
|
||||
detail = response.text
|
||||
raise RuntimeError(f"OpenRouter error {response.status_code}: {detail}")
|
||||
data = response.json()
|
||||
log_verbose("OpenRouter raw response", data)
|
||||
choices = data.get("choices")
|
||||
if not choices:
|
||||
raise RuntimeError("OpenRouter returned no choices")
|
||||
return choices[0]["message"]["content"].strip()
|
||||
|
||||
|
||||
def load_runbook() -> Dict[str, Dict[str, Any]]:
|
||||
"""Load runbook YAML into a dict keyed by rule_uid."""
|
||||
if not RUNBOOK_PATH.exists():
|
||||
raise FileNotFoundError(f"Runbook file not found: {RUNBOOK_PATH}")
|
||||
with RUNBOOK_PATH.open("r", encoding="utf-8") as handle:
|
||||
data = yaml.safe_load(handle) or {}
|
||||
alerts = data.get("alerts", [])
|
||||
index: Dict[str, Dict[str, Any]] = {}
|
||||
for entry in alerts:
|
||||
uid = entry.get("rule_uid")
|
||||
if uid:
|
||||
index[str(uid)] = entry
|
||||
LOGGER.info("Loaded %d runbook entries from %s", len(index), RUNBOOK_PATH)
|
||||
return index
|
||||
|
||||
|
||||
def _normalize_host_key(host: str) -> str:
|
||||
return host.strip().lower()
|
||||
|
||||
|
||||
def _parse_key_value_tokens(tokens: List[str]) -> Dict[str, str]:
|
||||
data: Dict[str, str] = {}
|
||||
for token in tokens:
|
||||
if "=" not in token:
|
||||
continue
|
||||
key, value = token.split("=", 1)
|
||||
data[key] = value
|
||||
return data
|
||||
|
||||
|
||||
def load_ansible_inventory() -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, str]]]:
|
||||
"""Parse a simple INI-style Ansible hosts file into host/group maps."""
|
||||
if not ANSIBLE_HOSTS_PATH.exists():
|
||||
LOGGER.warning("Ansible inventory not found at %s", ANSIBLE_HOSTS_PATH)
|
||||
return {}, {}
|
||||
hosts: Dict[str, Dict[str, Any]] = {}
|
||||
group_vars: Dict[str, Dict[str, str]] = {}
|
||||
current_group: Optional[str] = None
|
||||
current_section: str = "hosts"
|
||||
|
||||
with ANSIBLE_HOSTS_PATH.open("r", encoding="utf-8") as handle:
|
||||
for raw_line in handle:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if line.startswith("[") and line.endswith("]"):
|
||||
header = line[1:-1].strip()
|
||||
if ":" in header:
|
||||
group_name, suffix = header.split(":", 1)
|
||||
current_group = group_name
|
||||
current_section = suffix
|
||||
else:
|
||||
current_group = header
|
||||
current_section = "hosts"
|
||||
group_vars.setdefault(current_group, {})
|
||||
continue
|
||||
cleaned = line.split("#", 1)[0].strip()
|
||||
if not cleaned:
|
||||
continue
|
||||
tokens = shlex.split(cleaned)
|
||||
if not tokens:
|
||||
continue
|
||||
if current_section == "vars":
|
||||
vars_dict = _parse_key_value_tokens(tokens)
|
||||
group_vars.setdefault(current_group or "all", {}).update(vars_dict)
|
||||
continue
|
||||
host_token = tokens[0]
|
||||
host_key = _normalize_host_key(host_token)
|
||||
entry = hosts.setdefault(host_key, {"name": host_token, "definitions": [], "groups": set()})
|
||||
vars_dict = _parse_key_value_tokens(tokens[1:])
|
||||
entry["definitions"].append({"group": current_group, "vars": vars_dict})
|
||||
if current_group:
|
||||
entry["groups"].add(current_group)
|
||||
|
||||
LOGGER.info("Loaded %d Ansible inventory hosts from %s", len(hosts), ANSIBLE_HOSTS_PATH)
|
||||
return hosts, group_vars
|
||||
|
||||
|
||||
def _lookup_inventory(host: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
if not host:
|
||||
return None
|
||||
key = _normalize_host_key(host)
|
||||
entry = _INVENTORY_INDEX.get(key)
|
||||
if entry:
|
||||
return entry
|
||||
# try stripping domain suffix
|
||||
short = key.split(".", 1)[0]
|
||||
if short != key:
|
||||
return _INVENTORY_INDEX.get(short)
|
||||
return None
|
||||
|
||||
|
||||
def _merge_group_vars(groups: List[str], host_os: Optional[str]) -> Dict[str, str]:
|
||||
merged: Dict[str, str] = {}
|
||||
global_vars = _INVENTORY_GROUP_VARS.get("all")
|
||||
if global_vars:
|
||||
merged.update(global_vars)
|
||||
normalized_os = (host_os or "").lower()
|
||||
for group in groups:
|
||||
vars_dict = _INVENTORY_GROUP_VARS.get(group)
|
||||
if not vars_dict:
|
||||
continue
|
||||
connection = (vars_dict.get("ansible_connection") or "").lower()
|
||||
if connection == "winrm" and normalized_os == "linux":
|
||||
continue
|
||||
merged.update(vars_dict)
|
||||
return merged
|
||||
|
||||
|
||||
def _should_include_definition(group: Optional[str], vars_dict: Dict[str, str], host_os: Optional[str]) -> bool:
|
||||
if not vars_dict:
|
||||
return False
|
||||
normalized_os = (host_os or "").lower()
|
||||
connection = (vars_dict.get("ansible_connection") or "").lower()
|
||||
if connection == "winrm" and normalized_os != "windows":
|
||||
return False
|
||||
if connection == "local":
|
||||
return True
|
||||
if group and "windows" in group.lower() and normalized_os == "linux" and not connection:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def apply_inventory_context(context: Dict[str, Any]) -> None:
|
||||
"""Augment the alert context with SSH metadata from the Ansible inventory."""
|
||||
host = context.get("host")
|
||||
entry = _lookup_inventory(host)
|
||||
if not entry:
|
||||
return
|
||||
merged_vars = _merge_group_vars(list(entry.get("groups", [])), context.get("host_os"))
|
||||
for definition in entry.get("definitions", []):
|
||||
group_name = definition.get("group")
|
||||
vars_dict = definition.get("vars", {})
|
||||
if _should_include_definition(group_name, vars_dict, context.get("host_os")):
|
||||
merged_vars.update(vars_dict)
|
||||
ansible_host = merged_vars.get("ansible_host") or entry.get("name")
|
||||
ansible_user = merged_vars.get("ansible_user")
|
||||
ansible_port = merged_vars.get("ansible_port")
|
||||
ssh_common_args = merged_vars.get("ansible_ssh_common_args")
|
||||
ssh_key = merged_vars.get("ansible_ssh_private_key_file")
|
||||
connection = (merged_vars.get("ansible_connection") or "").lower()
|
||||
host_os = (context.get("host_os") or "").lower()
|
||||
if connection == "winrm" and host_os != "windows":
|
||||
for key in (
|
||||
"ansible_connection",
|
||||
"ansible_port",
|
||||
"ansible_password",
|
||||
"ansible_winrm_server_cert_validation",
|
||||
"ansible_winrm_scheme",
|
||||
):
|
||||
merged_vars.pop(key, None)
|
||||
connection = ""
|
||||
|
||||
context.setdefault("ssh_host", ansible_host or host)
|
||||
if ansible_user:
|
||||
context["ssh_user"] = ansible_user
|
||||
if ansible_port:
|
||||
context["ssh_port"] = ansible_port
|
||||
if ssh_common_args:
|
||||
context["ssh_common_args"] = ssh_common_args
|
||||
if ssh_key:
|
||||
context["ssh_identity_file"] = ssh_key
|
||||
context.setdefault("inventory_groups", list(entry.get("groups", [])))
|
||||
if connection == "local":
|
||||
context.setdefault("preferred_runner", "local")
|
||||
elif connection in {"", "ssh", "smart"}:
|
||||
context.setdefault("preferred_runner", "ssh")
|
||||
context.setdefault("inventory_groups", list(entry.get("groups", [])))
|
||||
|
||||
|
||||
def render_template(template: str, context: Dict[str, Any]) -> str:
|
||||
"""Very small mustache-style renderer for {{ var }} placeholders."""
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
key = match.group(1)
|
||||
return str(context.get(key, match.group(0)))
|
||||
|
||||
return _TEMPLATE_PATTERN.sub(replace, template)
|
||||
|
||||
|
||||
def extract_rule_uid(alert: Dict[str, Any], parent_payload: Dict[str, Any]) -> Optional[str]:
|
||||
"""Grafana webhooks may include rule UID in different fields."""
|
||||
candidates: List[Any] = [
|
||||
alert.get("ruleUid"),
|
||||
alert.get("rule_uid"),
|
||||
alert.get("ruleId"),
|
||||
alert.get("uid"),
|
||||
alert.get("labels", {}).get("rule_uid"),
|
||||
alert.get("labels", {}).get("ruleUid"),
|
||||
parent_payload.get("ruleUid"),
|
||||
parent_payload.get("rule_uid"),
|
||||
parent_payload.get("ruleId"),
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate:
|
||||
return str(candidate)
|
||||
# Fall back to Grafana URL parsing if present
|
||||
url = (
|
||||
alert.get("ruleUrl")
|
||||
or parent_payload.get("ruleUrl")
|
||||
or alert.get("generatorURL")
|
||||
or parent_payload.get("generatorURL")
|
||||
)
|
||||
if url and "/alerting/" in url:
|
||||
return url.rstrip("/").split("/")[-2]
|
||||
return None
|
||||
|
||||
|
||||
def derive_fallback_rule_uid(alert: Dict[str, Any], parent_payload: Dict[str, Any]) -> str:
|
||||
"""Construct a deterministic identifier when Grafana omits rule UIDs."""
|
||||
labels = alert.get("labels", {})
|
||||
candidates = [
|
||||
alert.get("fingerprint"),
|
||||
labels.get("alertname"),
|
||||
labels.get("host"),
|
||||
labels.get("instance"),
|
||||
parent_payload.get("groupKey"),
|
||||
parent_payload.get("title"),
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate:
|
||||
return str(candidate)
|
||||
return "unknown-alert"
|
||||
|
||||
|
||||
def build_fallback_runbook_entry(alert: Dict[str, Any], parent_payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Return a generic runbook entry so every alert can be processed."""
|
||||
labels = alert.get("labels", {})
|
||||
alertname = labels.get("alertname") or parent_payload.get("title") or "Grafana Alert"
|
||||
host = labels.get("host") or labels.get("instance") or "(unknown host)"
|
||||
return {
|
||||
"name": f"{alertname} (auto)",
|
||||
"llm_prompt": (
|
||||
"Grafana alert {{ alertname }} fired for {{ host }}.\n"
|
||||
"No dedicated runbook entry exists. Use the payload details, command outputs, "
|
||||
"and your own reasoning to propose likely causes, evidence to gather, and remediation steps."
|
||||
),
|
||||
"triage": [],
|
||||
"evidence_to_collect": [],
|
||||
"remediation": [],
|
||||
"metadata": {"host": host},
|
||||
}
|
||||
|
||||
|
||||
def summarize_dict(prefix: str, data: Optional[Dict[str, Any]]) -> str:
|
||||
if not data:
|
||||
return f"{prefix}: (none)"
|
||||
parts = ", ".join(f"{key}={value}" for key, value in sorted(data.items()))
|
||||
return f"{prefix}: {parts}"
|
||||
|
||||
|
||||
def determine_host_os(alert: Dict[str, Any]) -> str:
|
||||
"""Infer host operating system from labels or defaults."""
|
||||
labels = alert.get("labels", {})
|
||||
candidates = [
|
||||
labels.get("os"),
|
||||
labels.get("platform"),
|
||||
labels.get("system"),
|
||||
alert.get("os"),
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate:
|
||||
value = str(candidate).lower()
|
||||
if "win" in value:
|
||||
return "windows"
|
||||
if any(token in value for token in ("linux", "unix", "darwin")):
|
||||
return "linux"
|
||||
host = (labels.get("host") or labels.get("instance") or "").lower()
|
||||
if host.startswith("win") or host.endswith(".localdomain") and "win" in host:
|
||||
return "windows"
|
||||
inventory_os = infer_os_from_inventory(labels.get("host") or labels.get("instance"))
|
||||
if inventory_os:
|
||||
return inventory_os
|
||||
return TRIAGE_DEFAULT_OS
|
||||
|
||||
|
||||
def infer_os_from_inventory(host: Optional[str]) -> Optional[str]:
|
||||
if not host:
|
||||
return None
|
||||
entry = _lookup_inventory(host)
|
||||
if not entry:
|
||||
return None
|
||||
for definition in entry.get("definitions", []):
|
||||
vars_dict = definition.get("vars", {}) or {}
|
||||
connection = (vars_dict.get("ansible_connection") or "").lower()
|
||||
if connection == "winrm":
|
||||
return "windows"
|
||||
for group in entry.get("groups", []):
|
||||
if "windows" in (group or "").lower():
|
||||
return "windows"
|
||||
return None
|
||||
|
||||
|
||||
def truncate_text(text: str, limit: int = TRIAGE_OUTPUT_LIMIT) -> str:
|
||||
"""Trim long outputs to keep prompts manageable."""
|
||||
if not text:
|
||||
return ""
|
||||
cleaned = text.strip()
|
||||
if len(cleaned) <= limit:
|
||||
return cleaned
|
||||
return cleaned[:limit] + "... [truncated]"
|
||||
|
||||
|
||||
def gather_command_specs(entry: Dict[str, Any], host_os: str) -> List[Dict[str, Any]]:
|
||||
"""Collect command specs from triage steps and optional automation sections."""
|
||||
specs: List[Dict[str, Any]] = []
|
||||
for step in entry.get("triage", []):
|
||||
cmd = step.get(host_os)
|
||||
if not cmd:
|
||||
continue
|
||||
specs.append(
|
||||
{
|
||||
"summary": step.get("summary") or entry.get("name") or "triage",
|
||||
"shell": cmd,
|
||||
"runner": step.get("runner"),
|
||||
"os": host_os,
|
||||
}
|
||||
)
|
||||
for item in entry.get("automation_commands", []):
|
||||
target_os = item.get("os", host_os)
|
||||
if target_os and target_os.lower() != host_os:
|
||||
continue
|
||||
specs.append(item)
|
||||
if TRIAGE_MAX_COMMANDS > 0:
|
||||
return specs[:TRIAGE_MAX_COMMANDS]
|
||||
return specs
|
||||
|
||||
|
||||
def build_runner_command(
|
||||
rendered_command: str,
|
||||
runner: str,
|
||||
context: Dict[str, Any],
|
||||
spec: Dict[str, Any],
|
||||
) -> Tuple[Any, str, bool, str]:
|
||||
"""Return the subprocess args, display string, shell flag, and runner label."""
|
||||
runner = runner or TRIAGE_COMMAND_RUNNER
|
||||
runner = runner.lower()
|
||||
if runner == "ssh":
|
||||
host = spec.get("host") or context.get("ssh_host") or context.get("host")
|
||||
if not host:
|
||||
raise RuntimeError("Host not provided for ssh runner.")
|
||||
ssh_user = spec.get("ssh_user") or context.get("ssh_user") or TRIAGE_SSH_USER
|
||||
ssh_target = spec.get("ssh_target") or f"{ssh_user}@{host}"
|
||||
ssh_options = list(TRIAGE_SSH_OPTIONS)
|
||||
common_args = spec.get("ssh_common_args") or context.get("ssh_common_args")
|
||||
if common_args:
|
||||
ssh_options.extend(shlex.split(common_args))
|
||||
ssh_port = spec.get("ssh_port") or context.get("ssh_port")
|
||||
if ssh_port:
|
||||
ssh_options.extend(["-p", str(ssh_port)])
|
||||
identity_file = spec.get("ssh_identity_file") or context.get("ssh_identity_file")
|
||||
if identity_file:
|
||||
ssh_options.extend(["-i", identity_file])
|
||||
command_list = ["ssh", *ssh_options, ssh_target, rendered_command]
|
||||
display = " ".join(shlex.quote(part) for part in command_list)
|
||||
return command_list, display, False, "ssh"
|
||||
# default to local shell execution
|
||||
display = rendered_command
|
||||
return rendered_command, display, True, "local"
|
||||
|
||||
|
||||
def run_subprocess_command(
|
||||
command: Any,
|
||||
display: str,
|
||||
summary: str,
|
||||
use_shell: bool,
|
||||
runner_label: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute subprocess command and capture results."""
|
||||
LOGGER.info("Executing command (%s) via %s: %s", summary, runner_label, display)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=TRIAGE_COMMAND_TIMEOUT,
|
||||
shell=use_shell,
|
||||
check=False,
|
||||
)
|
||||
result = {
|
||||
"summary": summary,
|
||||
"command": display,
|
||||
"runner": runner_label,
|
||||
"exit_code": completed.returncode,
|
||||
"stdout": (completed.stdout or "").strip(),
|
||||
"stderr": (completed.stderr or "").strip(),
|
||||
"status": "ok" if completed.returncode == 0 else "failed",
|
||||
}
|
||||
log_verbose(f"Command result ({summary})", result)
|
||||
return result
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
result = {
|
||||
"summary": summary,
|
||||
"command": display,
|
||||
"runner": runner_label,
|
||||
"exit_code": None,
|
||||
"stdout": truncate_text((exc.stdout or "").strip()),
|
||||
"stderr": truncate_text((exc.stderr or "").strip()),
|
||||
"status": "timeout",
|
||||
"error": f"Command timed out after {TRIAGE_COMMAND_TIMEOUT}s",
|
||||
}
|
||||
log_verbose(f"Command timeout ({summary})", result)
|
||||
return result
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
LOGGER.exception("Command execution failed (%s): %s", summary, exc)
|
||||
result = {
|
||||
"summary": summary,
|
||||
"command": display,
|
||||
"runner": runner_label,
|
||||
"exit_code": None,
|
||||
"stdout": "",
|
||||
"stderr": "",
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
log_verbose(f"Command error ({summary})", result)
|
||||
return result
|
||||
|
||||
|
||||
def run_command_spec(spec: Dict[str, Any], context: Dict[str, Any]) -> Dict[str, Any]:
|
||||
summary = spec.get("summary") or spec.get("name") or "command"
|
||||
shell_cmd = spec.get("shell")
|
||||
if not shell_cmd:
|
||||
return {"summary": summary, "status": "skipped", "error": "No shell command provided."}
|
||||
rendered = render_template(shell_cmd, context)
|
||||
preferred_runner = context.get("preferred_runner")
|
||||
runner_choice = (spec.get("runner") or preferred_runner or TRIAGE_COMMAND_RUNNER).lower()
|
||||
try:
|
||||
command, display, use_shell, runner_label = build_runner_command(rendered, runner_choice, context, spec)
|
||||
except RuntimeError as exc:
|
||||
LOGGER.warning("Skipping command '%s': %s", summary, exc)
|
||||
return {"summary": summary, "status": "skipped", "error": str(exc), "command": rendered}
|
||||
return run_subprocess_command(command, display, summary, use_shell, runner_label)
|
||||
|
||||
|
||||
def execute_triage_commands(entry: Dict[str, Any], alert: Dict[str, Any], context: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
host_os = context.get("host_os") or determine_host_os(alert)
|
||||
context["host_os"] = host_os
|
||||
specs = gather_command_specs(entry, host_os)
|
||||
if not specs:
|
||||
LOGGER.info("No triage commands defined for host_os=%s", host_os)
|
||||
return []
|
||||
if not TRIAGE_ENABLE_COMMANDS:
|
||||
LOGGER.info("Command execution disabled; %d commands queued but skipped.", len(specs))
|
||||
return []
|
||||
LOGGER.info("Executing up to %d triage commands for host_os=%s", len(specs), host_os)
|
||||
results = []
|
||||
for spec in specs:
|
||||
results.append(run_command_spec(spec, context))
|
||||
return results
|
||||
|
||||
|
||||
def format_command_results_for_llm(results: List[Dict[str, Any]]) -> str:
|
||||
lines: List[str] = []
|
||||
for idx, result in enumerate(results, start=1):
|
||||
lines.append(f"{idx}. {result.get('summary')} [{result.get('status')}] {result.get('command')}")
|
||||
stdout = result.get("stdout")
|
||||
stderr = result.get("stderr")
|
||||
error = result.get("error")
|
||||
if stdout:
|
||||
lines.append(" stdout:")
|
||||
lines.append(indent(truncate_text(stdout), " "))
|
||||
if stderr:
|
||||
lines.append(" stderr:")
|
||||
lines.append(indent(truncate_text(stderr), " "))
|
||||
if error and result.get("status") != "ok":
|
||||
lines.append(f" error: {error}")
|
||||
if not lines:
|
||||
return "No command results were available."
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def parse_structured_response(text: str) -> Optional[Dict[str, Any]]:
|
||||
cleaned = text.strip()
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError:
|
||||
start = cleaned.find("{")
|
||||
end = cleaned.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
snippet = cleaned[start : end + 1]
|
||||
try:
|
||||
return json.loads(snippet)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def normalize_followup_command(item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"summary": item.get("summary") or item.get("name") or "Follow-up command",
|
||||
"shell": item.get("command") or item.get("shell"),
|
||||
"runner": item.get("runner"),
|
||||
"host": item.get("host") or item.get("target"),
|
||||
"ssh_user": item.get("ssh_user"),
|
||||
"os": (item.get("os") or item.get("platform") or "").lower() or None,
|
||||
}
|
||||
|
||||
|
||||
def investigate_with_langchain(
|
||||
entry: Dict[str, Any],
|
||||
alert: Dict[str, Any],
|
||||
parent_payload: Dict[str, Any],
|
||||
context: Dict[str, Any],
|
||||
initial_outputs: List[Dict[str, Any]],
|
||||
) -> Tuple[str, List[Dict[str, Any]]]:
|
||||
command_outputs = list(initial_outputs)
|
||||
prompt = build_prompt(entry, alert, parent_payload, context, command_outputs)
|
||||
log_verbose("Initial investigation prompt", prompt)
|
||||
if not OPENROUTER_API_KEY:
|
||||
return "OPENROUTER_API_KEY is not configured; unable to analyze alert.", command_outputs
|
||||
|
||||
llm = OpenRouterLLM(api_key=OPENROUTER_API_KEY, model_name=OPENROUTER_MODEL)
|
||||
dialogue = (
|
||||
prompt
|
||||
+ "\n\nRespond with JSON containing fields analysis, followup_commands, and complete. "
|
||||
"Request commands only when more evidence is required."
|
||||
)
|
||||
total_followup = 0
|
||||
final_summary = ""
|
||||
for iteration in range(TRIAGE_MAX_ITERATIONS):
|
||||
log_verbose(f"LLM dialogue iteration {iteration + 1}", dialogue)
|
||||
llm_text = llm(dialogue)
|
||||
log_verbose(f"LLM iteration {iteration + 1} output", llm_text)
|
||||
dialogue += f"\nAssistant:\n{llm_text}\n"
|
||||
parsed = parse_structured_response(llm_text)
|
||||
if parsed:
|
||||
log_verbose(f"LLM iteration {iteration + 1} parsed response", parsed)
|
||||
if not parsed:
|
||||
final_summary = llm_text
|
||||
break
|
||||
|
||||
analysis = parsed.get("analysis") or ""
|
||||
followups = parsed.get("followup_commands") or parsed.get("commands") or []
|
||||
final_summary = analysis
|
||||
complete_flag = bool(parsed.get("complete"))
|
||||
|
||||
if complete_flag or not followups:
|
||||
break
|
||||
|
||||
log_verbose(f"LLM iteration {iteration + 1} requested follow-ups", followups)
|
||||
allowed = max(0, TRIAGE_FOLLOWUP_MAX_COMMANDS - total_followup)
|
||||
if not TRIAGE_ENABLE_COMMANDS or allowed <= 0:
|
||||
dialogue += (
|
||||
"\nUser:\nCommand execution is disabled or budget exhausted. Provide final analysis with JSON format.\n"
|
||||
)
|
||||
continue
|
||||
|
||||
normalized_cmds: List[Dict[str, Any]] = []
|
||||
for raw in followups:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
normalized = normalize_followup_command(raw)
|
||||
if not normalized.get("shell"):
|
||||
continue
|
||||
cmd_os = normalized.get("os")
|
||||
if cmd_os and cmd_os != context.get("host_os"):
|
||||
continue
|
||||
normalized_cmds.append(normalized)
|
||||
|
||||
log_verbose(f"Normalized follow-up commands (iteration {iteration + 1})", normalized_cmds)
|
||||
if not normalized_cmds:
|
||||
dialogue += "\nUser:\nNo valid commands to run. Finalize analysis in JSON format.\n"
|
||||
continue
|
||||
|
||||
normalized_cmds = normalized_cmds[:allowed]
|
||||
executed_batch: List[Dict[str, Any]] = []
|
||||
for spec in normalized_cmds:
|
||||
executed = run_command_spec(spec, context)
|
||||
command_outputs.append(executed)
|
||||
executed_batch.append(executed)
|
||||
total_followup += 1
|
||||
|
||||
result_text = "Follow-up command results:\n" + format_command_results_for_llm(executed_batch)
|
||||
dialogue += (
|
||||
"\nUser:\n"
|
||||
+ result_text
|
||||
+ "\nUpdate your analysis and respond with JSON (analysis, followup_commands, complete).\n"
|
||||
)
|
||||
log_verbose("Executed follow-up commands", result_text)
|
||||
else:
|
||||
final_summary = final_summary or "Reached maximum iterations without a conclusive response."
|
||||
|
||||
if not final_summary:
|
||||
final_summary = "LLM did not return a valid analysis."
|
||||
|
||||
log_verbose("Final LLM summary", final_summary)
|
||||
return final_summary, command_outputs
|
||||
|
||||
|
||||
def build_context(alert: Dict[str, Any], parent_payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
labels = alert.get("labels", {})
|
||||
annotations = alert.get("annotations", {})
|
||||
context = {
|
||||
"alertname": labels.get("alertname") or alert.get("title") or parent_payload.get("title") or parent_payload.get("ruleName"),
|
||||
"host": labels.get("host") or labels.get("instance"),
|
||||
"iface": labels.get("interface"),
|
||||
"device": labels.get("device"),
|
||||
"vmid": labels.get("vmid"),
|
||||
"status": alert.get("status") or parent_payload.get("status"),
|
||||
"value": alert.get("value") or annotations.get("value"),
|
||||
"rule_url": alert.get("ruleUrl") or parent_payload.get("ruleUrl"),
|
||||
}
|
||||
context.setdefault("ssh_user", TRIAGE_SSH_USER)
|
||||
return context
|
||||
|
||||
|
||||
def build_prompt(
|
||||
entry: Dict[str, Any],
|
||||
alert: Dict[str, Any],
|
||||
parent_payload: Dict[str, Any],
|
||||
context: Dict[str, Any],
|
||||
command_outputs: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> str:
|
||||
template = entry.get("llm_prompt", "Alert {{ alertname }} fired for {{ host }}.")
|
||||
rendered_template = render_template(template, {k: v or "" for k, v in context.items()})
|
||||
|
||||
evidence = entry.get("evidence_to_collect", [])
|
||||
triage_steps = entry.get("triage", [])
|
||||
remediation = entry.get("remediation", [])
|
||||
|
||||
lines = [
|
||||
rendered_template.strip(),
|
||||
"",
|
||||
"Alert payload summary:",
|
||||
f"- Status: {context.get('status') or alert.get('status')}",
|
||||
f"- Host: {context.get('host')}",
|
||||
f"- Value: {context.get('value')}",
|
||||
f"- StartsAt: {alert.get('startsAt')}",
|
||||
f"- EndsAt: {alert.get('endsAt')}",
|
||||
f"- RuleURL: {context.get('rule_url')}",
|
||||
f"- Host OS (inferred): {context.get('host_os')}",
|
||||
"- Note: All timestamps are UTC/RFC3339 as provided by Grafana.",
|
||||
summarize_dict("- Labels", alert.get("labels")),
|
||||
summarize_dict("- Annotations", alert.get("annotations")),
|
||||
]
|
||||
|
||||
if evidence:
|
||||
lines.append("")
|
||||
lines.append("Evidence to gather (for automation reference):")
|
||||
for item in evidence:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
if triage_steps:
|
||||
lines.append("")
|
||||
lines.append("Suggested manual checks:")
|
||||
for step in triage_steps:
|
||||
summary = step.get("summary")
|
||||
linux = step.get("linux")
|
||||
windows = step.get("windows")
|
||||
lines.append(f"- {summary}")
|
||||
if linux:
|
||||
lines.append(f" Linux: {linux}")
|
||||
if windows:
|
||||
lines.append(f" Windows: {windows}")
|
||||
|
||||
if remediation:
|
||||
lines.append("")
|
||||
lines.append("Remediation ideas:")
|
||||
for item in remediation:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
if command_outputs:
|
||||
lines.append("")
|
||||
lines.append("Command execution results:")
|
||||
for result in command_outputs:
|
||||
status = result.get("status", "unknown")
|
||||
cmd_display = result.get("command", "")
|
||||
lines.append(f"- {result.get('summary')} [{status}] {cmd_display}")
|
||||
stdout = result.get("stdout")
|
||||
stderr = result.get("stderr")
|
||||
error = result.get("error")
|
||||
if stdout:
|
||||
lines.append(" stdout:")
|
||||
lines.append(indent(truncate_text(stdout), " "))
|
||||
if stderr:
|
||||
lines.append(" stderr:")
|
||||
lines.append(indent(truncate_text(stderr), " "))
|
||||
if error and status != "ok":
|
||||
lines.append(f" error: {error}")
|
||||
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def get_alerts(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
alerts = payload.get("alerts")
|
||||
if isinstance(alerts, list) and alerts:
|
||||
return alerts
|
||||
return [payload]
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def startup_event() -> None:
|
||||
global _RUNBOOK_INDEX, _INVENTORY_INDEX, _INVENTORY_GROUP_VARS
|
||||
_RUNBOOK_INDEX = load_runbook()
|
||||
_INVENTORY_INDEX, _INVENTORY_GROUP_VARS = load_ansible_inventory()
|
||||
LOGGER.info(
|
||||
"Alert webhook server ready with %d runbook entries and %d inventory hosts.",
|
||||
len(_RUNBOOK_INDEX),
|
||||
len(_INVENTORY_INDEX),
|
||||
)
|
||||
|
||||
|
||||
@app.post("/alerts")
|
||||
async def handle_alert(request: Request) -> Dict[str, Any]:
|
||||
payload = await request.json()
|
||||
LOGGER.info("Received Grafana payload: %s", json.dumps(payload, indent=2, sort_keys=True))
|
||||
results = []
|
||||
unmatched = []
|
||||
for alert in get_alerts(payload):
|
||||
LOGGER.info("Processing alert: %s", json.dumps(alert, indent=2, sort_keys=True))
|
||||
unmatched_reason: Optional[str] = None
|
||||
alert_status = str(alert.get("status") or payload.get("status") or "").lower()
|
||||
if alert_status and alert_status != "firing":
|
||||
details = {"reason": "non_firing_status", "status": alert_status, "alert": alert}
|
||||
unmatched.append(details)
|
||||
LOGGER.info("Skipping alert with status=%s (only 'firing' alerts are processed).", alert_status)
|
||||
continue
|
||||
rule_uid = extract_rule_uid(alert, payload)
|
||||
if not rule_uid:
|
||||
unmatched_reason = "missing_rule_uid"
|
||||
derived_uid = derive_fallback_rule_uid(alert, payload)
|
||||
details = {"reason": unmatched_reason, "derived_rule_uid": derived_uid, "alert": alert}
|
||||
unmatched.append(details)
|
||||
LOGGER.warning("Alert missing rule UID, using fallback identifier %s", derived_uid)
|
||||
rule_uid = derived_uid
|
||||
entry = _RUNBOOK_INDEX.get(rule_uid)
|
||||
runbook_matched = entry is not None
|
||||
if not entry:
|
||||
unmatched_reason = unmatched_reason or "no_runbook_entry"
|
||||
details = {"reason": unmatched_reason, "rule_uid": rule_uid, "alert": alert}
|
||||
unmatched.append(details)
|
||||
LOGGER.warning("No runbook entry for rule_uid=%s, using generic fallback.", rule_uid)
|
||||
entry = build_fallback_runbook_entry(alert, payload)
|
||||
context = build_context(alert, payload)
|
||||
context["host_os"] = determine_host_os(alert)
|
||||
context["rule_uid"] = rule_uid
|
||||
apply_inventory_context(context)
|
||||
initial_outputs = execute_triage_commands(entry, alert, context)
|
||||
try:
|
||||
llm_text, command_outputs = investigate_with_langchain(entry, alert, payload, context, initial_outputs)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
LOGGER.exception("Investigation failed for rule_uid=%s: %s", rule_uid, exc)
|
||||
raise HTTPException(status_code=502, detail=f"LLM investigation error: {exc}") from exc
|
||||
result = {
|
||||
"rule_uid": rule_uid,
|
||||
"alertname": entry.get("name"),
|
||||
"host": alert.get("labels", {}).get("host"),
|
||||
"llm_summary": llm_text,
|
||||
"command_results": command_outputs,
|
||||
"runbook_matched": runbook_matched,
|
||||
}
|
||||
if not runbook_matched and unmatched_reason:
|
||||
result["fallback_reason"] = unmatched_reason
|
||||
results.append(result)
|
||||
send_summary_email(alert, result, context)
|
||||
return {"processed": len(results), "results": results, "unmatched": unmatched}
|
||||
|
||||
|
||||
@app.post("/reload-runbook")
|
||||
def reload_runbook() -> Dict[str, Any]:
|
||||
global _RUNBOOK_INDEX, _INVENTORY_INDEX, _INVENTORY_GROUP_VARS
|
||||
_RUNBOOK_INDEX = load_runbook()
|
||||
_INVENTORY_INDEX, _INVENTORY_GROUP_VARS = load_ansible_inventory()
|
||||
return {"entries": len(_RUNBOOK_INDEX), "inventory_hosts": len(_INVENTORY_INDEX)}
|
||||
Reference in New Issue
Block a user