eval-set.schema.json

JSON · 60 lines · 4,406 bytes · wiki path 10 Architecture/contracts/eval-set.schema.json · download the raw file · cited from Contracts

Same folder: Connectors.Abstractions.cs · configuration-change.schema.json · configuration-input.schema.json · connector-capability.schema.json · execution-obligations.schema.json · ledger-record.schema.json · message-envelope.schema.json · model-turn.schema.json · module-manifest.schema.json · openapi.yaml · realtime-events.schema.json · runtime-state.schema.json · tool-package.schema.json · validate_case_journeys.py · validate_contracts.py · validate_runtime_contracts.py · validate_tool_packages.py · write-shape.schema.json

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://aisa.ablera.com/contracts/eval-set.schema.json",
  "title": "Eval set (platform seam 3 of 5)",
  "description": "Agent Framework § 6.5 (D66): one directory per case under the module's eval corpus — input/ (the arrival material), oracle/ (the reference outcome), tolerances.json (what 'match' means per dimension). Grading is deterministic first; only what cannot be checked mechanically goes to the isolated grader, one rubric dimension per call, with an explicit Unknown exit. This file is the set's manifest (eval-set.json at the corpus root); each case directory carries its own tolerances.json (#/$defs/tolerances).",
  "type": "object",
  "additionalProperties": false,
  "required": ["eval_key", "version", "module", "owner_profile_key", "cases", "grader_profile_key", "run_triggers"],
  "properties": {
    "eval_key": { "type": "string", "pattern": "^[a-z][a-z0-9_.\\-]+$" },
    "version": { "type": "integer", "minimum": 1 },
    "module": { "type": "string", "enum": ["configuration", "support", "source", "platform"] },
    "owner_profile_key": { "type": "string" },
    "corpus_kind": { "type": "string", "enum": ["round_trip", "resolved_cases", "merged_changes", "replay", "canary"], "description": "Configuration = round_trip; Support = resolved_cases; Development = merged_changes (a lineage pair is one case)" },
    "grader_profile_key": { "type": "string", "const": "platform.grader" },
    "run_triggers": { "type": "array", "minItems": 1, "items": { "type": "string", "enum": ["publish", "domain_change", "skill_change", "route_change", "schedule", "release"] } },
    "budget_minutes": { "type": "number", "minimum": 0, "description": "the set's own time budget per run; a set whose budget exceeds the errors it catches is trimmed" },
    "cases": {
      "type": "array", "minItems": 1,
      "items": {
        "type": "object", "additionalProperties": false,
        "required": ["case_key", "input", "oracle", "tolerances"],
        "properties": {
          "case_key": { "type": "string" },
          "source": { "type": "string", "description": "where the case came from: the v1 ticket, the product code, the merged commit" },
          "input": { "type": "string", "pattern": "^input/", "description": "directory: ticket snapshot · spec artefact · requirement text; identifiers are handles" },
          "oracle": { "type": "string", "pattern": "^oracle/", "description": "directory: mechanism + fix shape · exported product + counts · merged commit + test evidence" },
          "tolerances": { "type": "string", "pattern": "tolerances\\.json$" },
          "lineage_pair": { "type": "object", "properties": { "customer_branch": { "type": "string" }, "master_port": { "type": "string" }, "no_counterpart_reason": { "type": "string" } } }
        }
      }
    }
  },
  "$defs": {
    "tolerances": {
      "title": "tolerances.json — one per case directory",
      "type": "object",
      "additionalProperties": false,
      "required": ["dimensions"],
      "properties": {
        "dimensions": {
          "type": "array", "minItems": 1,
          "items": {
            "type": "object", "additionalProperties": false,
            "required": ["name", "check"],
            "properties": {
              "name": { "type": "string", "description": "completeness · counts · correctness · excess · semantics · mechanism_match · fix_safety · flagged_missing · placement · diff_shape · test_coverage · lineage_handling" },
              "check": { "type": "string", "enum": ["exact_count", "status_equality", "diff_zero", "set_equality", "numeric_within", "rubric"], "description": "the first five are deterministic and run without a model; rubric goes to the isolated grader" },
              "tolerance": { "type": "number", "minimum": 0, "description": "numeric_within only" },
              "rubric": { "type": "string", "description": "rubric only: the one question the grader answers with a score 0–1 or Unknown" },
              "weight": { "type": "number", "minimum": 0, "maximum": 1 },
              "blocking": { "type": "boolean", "default": true, "description": "a failed blocking dimension makes the case red" }
            }
          }
        },
        "pass_threshold": { "type": "number", "minimum": 0, "maximum": 1, "default": 1 }
      }
    }
  }
}