#!/usr/bin/env python3
"""Measure a local payload using a named tiktoken encoding; no model requests."""
import argparse
import hashlib
import importlib.metadata
import json
import os
from pathlib import Path
import tempfile


def measure(path, encoding, cache_path=None, include_ids=False):
    import tiktoken
    if Path(path).stat().st_size > 2_100_000:
        raise ValueError("Payload exceeds the 2.1 MB local limit.")
    raw = Path(path).read_bytes()
    text = raw.decode("utf-8")
    version = importlib.metadata.version("tiktoken")
    digest = hashlib.sha256(raw).hexdigest()
    key = f"{version}:{encoding}:{digest}"
    cache = {}
    if cache_path and Path(cache_path).exists():
        if Path(cache_path).stat().st_size > 20_000_000:
            raise ValueError("Token cache exceeds 20 MB; choose a new cache file.")
        cache = json.loads(Path(cache_path).read_text())
        if cache.get("schemaVersion") != "1.0" or not isinstance(cache.get("entries"), dict):
            raise ValueError("Unsupported token cache.")
    entries = cache.setdefault("entries", {})
    entry = entries.get(key)
    # Counts-only cache entries cannot fulfill an explicit token-ID request.
    reuse = isinstance(entry, dict) and entry.get("sha256") == digest and isinstance(entry.get("tokens"), int) and entry["tokens"] >= 0 and (not include_ids or isinstance(entry.get("ids"), list))
    if not reuse:
        enc = tiktoken.get_encoding(encoding)
        ids = enc.encode_ordinary(text)  # Source special-marker strings remain ordinary data.
        entry = {"sha256": digest, "tokens": len(ids), **({"ids": ids} if include_ids else {})}
        entries[key] = entry
        # A bounded cache; no original text or filesystem paths are stored.
        while len(entries) > 20:
            del entries[next(iter(entries))]
        if cache_path:
            cache["schemaVersion"] = "1.0"
            while len(json.dumps(cache)) > 20_000_000 and len(entries) > 1:
                del entries[next(iter(entries))]
            if len(json.dumps(cache)) > 20_000_000:
                raise ValueError("Token-ID cache exceeds 20 MB; use counts only.")
            target = Path(cache_path)
            with tempfile.NamedTemporaryFile("w", dir=target.parent, delete=False) as tmp:
                temporary = tmp.name
                json.dump(cache, tmp, ensure_ascii=False, separators=(",", ":"))
            try:
                os.chmod(temporary, 0o600)
                os.replace(temporary, target)
            finally:
                if os.path.exists(temporary):
                    os.unlink(temporary)
    return {"schemaVersion": "1.0", "encoding": encoding, "tokenizer": f"tiktoken/{version}", "sha256": digest,
            "utf8Bytes": len(raw), "tokens": entry["tokens"], "cacheHit": bool(reuse),
            **({"ids": entry["ids"]} if include_ids else {}),
            "scope": "Exact named-encoding count of this complete UTF-8 file, including JSON wrappers and whitespace. Not a universal model count, billed prompt count or dollar saving. Pretokenizing identical text does not reduce its token count. IDs are encoding-specific; do not paste numeric IDs into a text prompt. Cache reuse saves local tokenization work only. No model, archive or source request."}


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("file")
    parser.add_argument("--encoding", choices=["o200k_base", "cl100k_base"], default="o200k_base")
    parser.add_argument("--cache")
    parser.add_argument("--include-ids", action="store_true")
    args = parser.parse_args()
    try:
        print(json.dumps(measure(args.file, args.encoding, args.cache, args.include_ids), ensure_ascii=False))
    except ImportError:
        parser.exit(1, "Install tiktoken in your environment: python3 -m pip install tiktoken\n")
    except (ValueError, OSError, KeyError) as error:
        parser.exit(1, str(error) + "\n")
