{
  "conventions": {
    "q_learning": "target = r + gamma max over every action in available_actions[s_next] of Q(s_next,a) for nonterminal transitions",
    "return": "G_t = R_(t+1) + gamma R_(t+2) + ...; no reward follows the terminal transition",
    "reward_indexing": "trajectory rewards are [R1, R2, R3]; returns are [G0, G1, G2], so returns[t] begins with rewards[t] = R_(t+1)",
    "rounding": "IEEE-754 binary64 calculations remain unrounded; only emitted derived fields are rounded to 9 decimal places with Python round (ties-to-even)",
    "terminal_bootstrap": "zero; the terminal available-action set is empty and no maximum is taken",
    "trajectory_semantics": "trajectory lists actions actually sampled in this episode; available_actions separately and exhaustively defines the fixture action set at every state"
  },
  "creator": "reinforcement-learning-editor",
  "episode_fixture": {
    "available_actions": {
      "bridge": [
        "advance"
      ],
      "goal": [
        "advance",
        "safe",
        "greedy"
      ],
      "start": [
        "advance"
      ],
      "terminal": []
    },
    "gamma": 0.9,
    "returns": [
      4.85,
      6.5,
      5.0
    ],
    "rewards": [
      -1,
      2,
      5
    ],
    "states": [
      "start",
      "bridge",
      "goal",
      "terminal"
    ],
    "trajectory": [
      {
        "action": "advance",
        "next_state": "bridge",
        "reward": -1,
        "state": "start",
        "terminal": false
      },
      {
        "action": "advance",
        "next_state": "goal",
        "reward": 2,
        "state": "bridge",
        "terminal": false
      },
      {
        "action": "advance",
        "next_state": "terminal",
        "reward": 5,
        "state": "goal",
        "terminal": true
      }
    ]
  },
  "license": "CC BY 4.0",
  "license_url": "https://creativecommons.org/licenses/by/4.0/",
  "policy_contrast": {
    "description": "Counterfactual branch for the bridge transition, separate from the recorded advance trajectory: if SARSA next samples safe it uses value 2, while Q-learning uses the maximum 4.",
    "q_learning_updated_q": 3.3,
    "relation_to_episode_trajectory": "separate counterfactual update; does not assert that the recorded episode sampled safe",
    "sarsa_behavior_action": "safe",
    "sarsa_update": {
      "alpha": 0.5,
      "bootstrap_value": 2.0,
      "gamma": 0.9,
      "initial_q": 1.0,
      "reward": 2,
      "target": 3.8,
      "td_error": 2.8,
      "terminal": false,
      "updated_q": 2.4
    }
  },
  "policy_gradient_update": {
    "action_order": [
      "left",
      "right"
    ],
    "alpha": 0.1,
    "environment": "one nonterminal state with actions left/right; the sampled right action ends the episode with return 2",
    "gamma_power": 1.0,
    "gradient_estimate": [
      -1.0,
      1.0
    ],
    "logits_after": [
      -0.1,
      0.1
    ],
    "logits_before": [
      0.0,
      0.0
    ],
    "parameterization": "two-logit softmax",
    "probabilities_after": [
      0.450166003,
      0.549833997
    ],
    "probabilities_before": [
      0.5,
      0.5
    ],
    "return": 2.0,
    "sampled_action": "right",
    "score_gradient_log_pi_sampled_action": [
      -0.5,
      0.5
    ]
  },
  "provenance": "Original fictional episode and TD/policy-update fixture; no people or deployment data.",
  "q_learning_update": {
    "alpha": 0.5,
    "bootstrap_value": 4.0,
    "gamma": 0.9,
    "initial_q": 1.0,
    "next_state": "goal",
    "next_state_action_values": {
      "advance": 3.0,
      "greedy": 4.0,
      "safe": 2.0
    },
    "reward": 2,
    "selected_bootstrap_action": "greedy",
    "state_action": [
      "bridge",
      "advance"
    ],
    "target": 5.6,
    "td_error": 4.6,
    "terminal": false,
    "updated_q": 3.3
  },
  "q_value_snapshot": {
    "goal": {
      "advance": 3.0,
      "greedy": 4.0,
      "safe": 2.0
    }
  },
  "schema": 2,
  "source": "AI Encyclopedia CN, Chapter 13",
  "terminal_update": {
    "alpha": 0.5,
    "bootstrap_value": 0.0,
    "gamma": 0.9,
    "initial_q": 1.0,
    "next_state": "terminal",
    "note": "Bootstrap is zero because this transition is terminal; no max over an empty action set is evaluated.",
    "reward": 5,
    "state_action": [
      "goal",
      "advance"
    ],
    "stored_next_value_ignored": 999.0,
    "target": 5.0,
    "td_error": 4.0,
    "terminal": true,
    "updated_q": 3.0
  }
}