📄 context.py
/home/palash/git/local-ai/server/features/context.py
Language: py • Lines: 321
"""Token estimation, context trimming and context compaction."""

import base64
import json
import os
import re

import requests

from server.features.state import M


def strip_html(text):
    text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL)
    text = re.sub(r"<script[^>]*>.*?</script>", "", text, flags=re.DOTALL)
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"\s+", " ", text).strip()
    return text


def _text_tokens(s):
    if not s:
        return 0
    # Multilingual/non-ASCII characters eat up far more tokens (~2 chars per token vs ~4 for English)
    non_ascii = sum(1 for ch in s if ord(ch) > 0x7F)
    divisor = 2.0 if non_ascii > len(s) * 0.15 else 4.0
    return int(len(s) / divisor)


def estimate_tokens(messages, include_tools=True):
    total = M.TOOLS_TOKEN_COST if include_tools else 0

    for msg in messages:
        total += M.PER_MESSAGE_OVERHEAD
        content = msg.get("content", "")

        # Standard string content
        if isinstance(content, str):
            total += _text_tokens(content)

        # Multi-modal array content (text + images + audio)
        elif isinstance(content, list):
            for part in content:
                if not isinstance(part, dict):
                    continue
                ptype = part.get("type")
                if ptype == "text":
                    total += _text_tokens(part.get("text", ""))
                elif ptype in ("image_url", "input_image", "image"):
                    total += M.IMAGE_TOKEN_COST
                elif ptype in ("audio_url", "input_audio", "audio"):
                    total += M.AUDIO_TOKEN_COST

        # Tool call tokens
        for tc in msg.get("tool_calls") or []:
            total += _text_tokens(json.dumps(tc))

    # Return MUST be outside the for-loop!
    return max(1, total)


def trim_messages_for_context(messages):
    trimmed = list(messages)
    sys_msg = None
    if trimmed and trimmed[0].get("role") == "system":
        sys_msg = trimmed.pop(0)
    while estimate_tokens(trimmed) > M.MAX_INPUT_TOKENS and len(trimmed) > 1:
        trimmed.pop(0)
    if sys_msg:
        trimmed.insert(0, sys_msg)
    return trimmed


def _summarize_with_llm(text, mode="gpu"):
    payload = {
        "model": M.server_model_id(mode),
        "messages": [
            {
                "role": "system",
                "content": "You summarize conversations concisely, preserving key facts, decisions, user preferences, and unresolved questions.",
            },
            {"role": "user", "content": text},
        ],
        "max_tokens": 1024,
        "temperature": 0.3,
        "stream": False,
    }
    try:
        M.mark_slot_kv_dirty(mode)
        r = requests.post(M.server_url(mode), json=payload, timeout=120)
        r.raise_for_status()
        return r.json()["choices"][0]["message"]["content"]
    except Exception as e:
        print(f"[compact] LLM summarization failed: {e}")
        return None


def compact_messages_copy(messages, keep_messages=6, mode="gpu"):
    """Return a compacted COPY of the message list (summary + recent messages)
    WITHOUT modifying the stored session. Old messages are summarized, not deleted."""
    msgs = list(messages)
    sys_msg = None
    if msgs and msgs[0].get("role") == "system":
        sys_msg = msgs.pop(0)
    if len(msgs) <= keep_messages + 1:
        return ([sys_msg] + msgs) if sys_msg else msgs
    to_compact = msgs[:-keep_messages] if keep_messages > 0 else msgs
    recent = msgs[-keep_messages:] if keep_messages > 0 else []
    compact_text = ""
    for m in to_compact:
        role = m.get("role", "unknown")
        content = m.get("content", "")
        if isinstance(content, list):
            parts = []
            for p in content:
                if isinstance(p, dict):
                    if p.get("type") == "text":
                        parts.append(p.get("text", ""))
            content = " ".join(parts)
        if not content:
            continue
        compact_text += f"[{role}]: {content}\n\n"
    if not compact_text.strip():
        return ([sys_msg] + msgs) if sys_msg else msgs
    summary = M._summarize_with_llm(
        f"Compress the following conversation into a short paragraph, keeping all important details:\n\n{compact_text}",
        mode,
    )
    if summary is None:
        return ([sys_msg] + msgs) if sys_msg else msgs
    new_msgs = []
    if sys_msg:
        new_msgs.append(sys_msg)
    new_msgs.append({"role": "system", "content": f"[Compressed context]: {summary}"})
    new_msgs.extend(recent)
    return new_msgs


def sanitize_content_for_llm(messages):
    """Return a COPY of ``messages`` with content parts the LLM backend cannot
    process (e.g. ``audio_url``) removed, so a stale multimodal message can't
    make llama-server reject the whole request (HTTP 400 "unsupported
    content[].type"). The stored session is left untouched.
    """
    sanitized = []
    for msg in messages:
        content = msg.get("content")
        if not isinstance(content, list):
            sanitized.append(msg)
            continue
        parts = []
        dropped_audio = False
        for p in content:
            if not isinstance(p, dict):
                continue
            if p.get("type") in ("audio_url", "input_audio", "audio"):
                dropped_audio = True
                continue
            parts.append(p)
        if dropped_audio:
            parts.append(
                {
                    "type": "text",
                    "text": "[Voice message omitted — audio input is not supported by this model]",
                }
            )
        sanitized.append({**msg, "content": parts})
    return sanitized


def resolve_image_path(url):
    """Resolve a ``/uploads/`` or ``/output/`` URL to a local file path, or
    ``None`` if the URL is unknown or the file is missing."""
    if not url:
        return None
    fpath = None
    if url.startswith("/uploads/"):
        fname = os.path.basename(url.split("?", 1)[0])
        fpath = os.path.join(M.UPLOADS_DIR, fname)
    elif url.startswith("/output/"):
        rel = url[len("/output/"):].split("?", 1)[0]
        fpath = os.path.join(M.IMG_PATH, rel)
    if not fpath or not os.path.isfile(fpath):
        return None
    return fpath


def _image_to_data_url(url):
    """Resolve an image URL the server can serve to a ``data:`` URL.

    Understands ``/uploads/`` (user uploads) and ``/output/`` (generated
    images) so those bytes can be embedded in the LLM request. ``data:`` URLs
    pass through unchanged; anything unknown returns ``None``.
    """
    if not url:
        return None
    if url.startswith("data:"):
        return url
    fpath = resolve_image_path(url)
    if not fpath:
        return None
    ext = os.path.splitext(fpath)[1].lower()
    mime = {
        ".png": "image/png",
        ".jpg": "image/jpeg",
        ".jpeg": "image/jpeg",
        ".webp": "image/webp",
        ".gif": "image/gif",
    }.get(ext, "image/jpeg")
    try:
        with open(fpath, "rb") as f:
            raw = f.read()
    except OSError:
        return None
    return f"data:{mime};base64,{base64.b64encode(raw).decode()}"


def _reference_historical_images(messages):
    """Return a COPY of ``messages`` where ``image_url`` parts are kept lean.

    The image attached to the most recent user message stays visible (its bytes
    are embedded via ``_image_to_data_url``), preserving the current turn's
    vision. Every older image becomes a compact text marker carrying the file
    URL — the model can call ``read_image`` to actually view one, so history
    never drags megabytes of base64 through the context window.
    """
    last_user_idx = -1
    for i, m in enumerate(messages):
        if m.get("role") == "user":
            last_user_idx = i

    out = []
    for i, m in enumerate(messages):
        content = m.get("content")
        if not isinstance(content, list):
            out.append(m)
            continue
        parts = []
        for p in content:
            if isinstance(p, dict) and p.get("type") == "image_url":
                url = p.get("image_url", {}).get("url", "")
                if i == last_user_idx:
                    data_url = _image_to_data_url(url)
                    if data_url:
                        parts.append({"type": "image_url", "image_url": {"url": data_url}})
                        continue
                    parts.append({"type": "text", "text": f"[IMAGE: {url}]"})
                else:
                    parts.append(
                        {
                            "type": "text",
                            "text": f"[IMAGE: {url} — use the read_image tool to view this image]",
                        }
                    )
            else:
                parts.append(p)
        out.append({**m, "content": parts})
    return out


def _latest_read_image_url(messages):
    """Return the image URL of the most recent successful ``read_image`` tool
    result in ``messages`` (or ``None``)."""
    for m in reversed(messages):
        if m.get("role") != "tool":
            continue
        content = m.get("content")
        if not isinstance(content, str):
            continue
        try:
            data = json.loads(content)
        except (TypeError, ValueError):
            continue
        if data.get("ok") is True and data.get("image_url"):
            return data["image_url"]
    return None


def prepare_context_for_llm(sid, messages, mode="gpu"):
    """Build the message list to send to the LLM. When the conversation nears the
    context limit, old messages are summarized into a compressed context block —
    but the stored session is left untouched, so no messages are deleted.
    Historical images are referenced by path (see ``read_image``) instead of
    being re-sent as base64 on every round."""
    messages = sanitize_content_for_llm(messages)
    messages = _reference_historical_images(messages)
    total = estimate_tokens(messages)
    if total <= M.AUTO_COMPACT_THRESHOLD:
        context = trim_messages_for_context(messages)
        with M._effective_contexts_lock:
            M._effective_contexts.pop(sid, None)
        return context
    print(f"[context] Session {sid} estimate {total} tokens exceeds threshold {M.AUTO_COMPACT_THRESHOLD}; building compressed context for LLM")
    compacted = compact_messages_copy(messages, mode=mode)
    context = trim_messages_for_context(compacted)
    print(f"[context] Compressed context built; estimate after: {estimate_tokens(context)}")
    with M._effective_contexts_lock:
        M._effective_contexts[sid] = context
    return context


def effective_token_estimate(sid, messages):
    """Report the token count the UI shows: the compressed context actually sent
    to the LLM once compression has kicked in, falling back to the full history."""
    with M._effective_contexts_lock:
        cached = M._effective_contexts.get(sid)
    if cached is not None:
        return estimate_tokens(cached)
    return estimate_tokens(messages)


def context_token_report(sid, messages):
    """Token report for the UI: effective count sent to the LLM, the raw stored
    count, and whether context compression is currently active."""
    effective = effective_token_estimate(sid, messages)
    raw = estimate_tokens(messages)
    return {
        "token_estimate": effective,
        "raw_token_estimate": raw,
        "context_compressed": raw > effective,
    }