{
  "provenance": {
    "source": "Metkagram",
    "source_url": "https://metkagram.github.io",
    "source_repository": "https://github.com/metkagram/metkagram.github.io",
    "creator": "Metkagram",
    "creator_url": "https://github.com/metkagram",
    "maintainer": "Applied Systems Lab at MetalHatsCats",
    "maintainer_url": "https://metalhatscats.com",
    "license": "Metkagram Source-Available Terms",
    "license_url": "https://metkagram.github.io/en/licensing/",
    "rights_status": "source-available-not-open-source",
    "rights_url": "https://metkagram.github.io/rights.json",
    "historical_license": "CC BY-NC 4.0 before 2026-08-17; prior grants remain valid for copies received under those terms",
    "attribution_required": true,
    "attribution_text": "Source: Metkagram — https://metkagram.github.io/",
    "attribution_html": "Source: <a href=\"https://metkagram.github.io/\">Metkagram</a>",
    "terms_url": "https://metkagram.github.io/en/licensing/",
    "privacy_url": "https://metkagram.github.io/en/legal/privacy/",
    "contact_email": "metalhatscats@gmail.com",
    "contact_url": "https://www.linkedin.com/company/metalhatscats",
    "collaboration_url": "https://metkagram.github.io/en/licensing/",
    "dataset_version": "1.0.0+5b4307882dc4",
    "release_date": "2026-08-19",
    "canonical_url": "https://metkagram.github.io/api/v1/evals/reasoning-routing.json",
    "record_type": "retrieval_benchmark_manifest",
    "record_id": "reasoning-routing-v1",
    "content_hash": "e734c9334a34c310cc7a6c69c2afbc36fea82bac1684b18e1b97d86f373e72ec"
  },
  "data": {
    "schemaVersion": 1,
    "id": "reasoning-routing-v1",
    "title": "Metkagram Intent-to-Pattern Retrieval Benchmark",
    "datasetVersion": "1.0.0+5b4307882dc4",
    "releaseDate": "2026-08-19",
    "caseCount": 54,
    "locales": {
      "en": 36,
      "ru": 18
    },
    "splits": {
      "core": 36,
      "near_neighbor": 18
    },
    "task": "Route a natural-language communicative goal to a Metkagram intent, reasoning move and up to three canonical pattern IDs.",
    "metrics": {
      "intentTop1": "Expected intent is ranked first.",
      "moveTop1": "Expected reasoning move is ranked first.",
      "patternHitAt3": "At least one editorially acceptable canonical pattern appears in the first three pattern results."
    },
    "bundledBaseline": {
      "type": "deterministic editorial resolver",
      "pass": true,
      "metrics": {
        "cases": 54,
        "intent_top1_accuracy": 1,
        "intent_recall_at3": 1,
        "move_top1_accuracy": 1,
        "pattern_hit_at1": 1,
        "pattern_hit_at3": 1,
        "intent_mrr": 1,
        "pattern_mrr": 1
      },
      "report": "https://metkagram.github.io/data/reasoning-evaluation.json"
    },
    "assets": {
      "benchmarkJson": "https://metkagram.github.io/data/reasoning-benchmark.json",
      "tasksJsonl": "https://metkagram.github.io/evals/reasoning-routing/tasks.jsonl",
      "baselineReport": "https://metkagram.github.io/data/reasoning-evaluation.json",
      "protocol": "https://metkagram.github.io/evals/reasoning-routing/README.md",
      "pages": {
        "en": "https://metkagram.github.io/en/evals/",
        "ru": "https://metkagram.github.io/ru/evals/"
      }
    },
    "limitations": [
      "The cases, taxonomy, acceptable answers and bundled resolver are maintained by the same project.",
      "The bundled score is an internal editorial regression signal, not independent external validation.",
      "The benchmark does not measure learning efficacy, general writing quality or overall language-model capability.",
      "Public gold labels make the suite reproducible but unsuitable as a hidden leaderboard test without an independently held-out extension."
    ],
    "reportingRequirements": [
      "Record the benchmark dataset version and run date.",
      "Record the system/model version and prompt or retrieval configuration.",
      "State whether the system had access to the Metkagram taxonomy, API or corpus.",
      "Report intent top-1, move top-1 and pattern hit@3 together, plus misses or abstentions."
    ],
    "rights": {
      "label": "Metkagram Source-Available Terms",
      "terms": "https://metkagram.github.io/en/licensing/",
      "attribution": "Source: Metkagram — https://metkagram.github.io/"
    }
  }
}