[
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Quelmithin",
  "n71_answer": "Quelmithin",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Crysthene Pool",
  "n71_answer": "Crysthene Pool",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "tutoring session (weekly)",
  "n71_answer": "tutoring session (weekly)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "partner",
  "question": "What's my partner's name?",
  "gold_answer": "James",
  "n71_answer": "James",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Therwyn Compact, Xylorim Scooter",
  "n71_answer": "Zyvanta Sedan, Therwyn Compact, Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "multivitamin",
  "n71_answer": "multivitamin",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Velthari Studio",
  "n71_answer": "Velthari Studio",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "Uncertain — previously 'tutoring session (weekly)', but health_condition changed",
  "n71_answer": "trivia night Wednesdays",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "partner",
  "question": "What's my partner's name?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is pottery, plays rock climbing, and is part of book club.",
  "n71_answer": "pottery, rock climbing",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-001",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "1.5 hours",
  "n71_answer": "1.5 hours",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Zefiro's Kitchen",
  "n71_answer": "Zefiro's Kitchen",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "varies by shift",
  "n71_answer": "varies by shift",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Therwyn Compact, Dranthis Pickup, Xylorim Scooter",
  "n71_answer": "Therwyn Compact, Dranthis Pickup, Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "around 10 min",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Uncertain — previously 'Zefiro's Kitchen', but residence_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none — explicitly removed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is knitting, plays basketball, and is part of board game group.",
  "n71_answer": "knitting, basketball, reading books, potluck dinner group",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-002",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Zorathel Peak Gym",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "30 min",
  "n71_answer": "30 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "food_allergy",
  "question": "What food allergies do I have?",
  "gold_answer": "eggs",
  "n71_answer": "eggs",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "partner",
  "question": "List all partners I've had over time, from earliest to latest.",
  "gold_answer": "Sarah, Emily, Alex",
  "n71_answer": "Sarah, Alex, Emily",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Kaelstrom Fitness",
  "n71_answer": "Kaelstrom Fitness",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "Uncertain — previously '30 min', but residence_location changed",
  "n71_answer": "30 min",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "food_allergy",
  "question": "What food allergies do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is guitar, plays rock climbing, and is part of hiking club.",
  "n71_answer": "guitar, photography, camping, hiking, rock climbing",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-003",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "no travel planned",
  "n71_answer": "European Road Trip 2023 from London to Nice and back via Harwich, Amsterdam, Lake Como, Lyon, Reims, Calais, and Eurotunnel.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "alone",
  "n71_answer": "Alone",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "trivia night Wednesdays",
  "n71_answer": "trivia night on Wednesdays",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Therwyn Compact, Quorvel EV, Xylorim Scooter",
  "n71_answer": "Quorvel EV, Xylorim Scooter",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "island getaway",
  "n71_answer": "European Road Trip 2023 from London to Nice and back via Harwich, Hook of Holland, Amsterdam, Luxembourg City, Lucerne, Lake Como, Lyon, Reims, Calais, and returning to London by April 16th.",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "Uncertain — previously 'alone', but relationship_status changed",
  "n71_answer": "You don't have that information.",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "trivia night on Wednesdays",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is guitar, plays pickleball, and is part of hiking club.",
  "n71_answer": "guitar, pickleball, trivia night on Wednesdays, plant propagation, orchids",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-004",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "paying off credit card debt",
  "n71_answer": "paying off credit card debt",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "Python programming",
  "n71_answer": "Python programming",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "engagement ring",
  "n71_answer": "engagement ring",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "The Veldrakon Spiral (TV), Whispers of Zarinthea (TV), The Quorath Codex (book)",
  "n71_answer": "The Veldrakon Spiral (TV), Whispers of Zarinthea (TV), The Quorath Codex (book)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "saving for kids' education fund",
  "n71_answer": "saving for kids' education fund",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "Uncertain — previously 'Python programming', but school changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "engagement ring",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is knitting, plays basketball, and is part of hiking club.",
  "n71_answer": "knitting, basketball, making homemade pizza",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-005",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dermatologist (monthly)",
  "n71_answer": "dermatologist (monthly)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Zorathel Peak Gym",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Dyntheral",
  "n71_answer": "Dyntheral",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "Spanish",
  "n71_answer": "Spanish",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "Whispers of Zarinthea (TV), Thyrelion (anime), a history documentary series",
  "n71_answer": "Thyrelion (anime), history documentary series",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "physical therapy (2x/week)",
  "n71_answer": "physical therapy (2x/week)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Kaelstrom Fitness",
  "n71_answer": "Kaelstrom Fitness",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Uncertain — previously 'Dyntheral', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Spanish",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays soccer, and is part of volunteer group.",
  "n71_answer": "woodworking, soccer, hiking, pickup basketball on Saturdays, watching history documentary series",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-006",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dermatologist (monthly)",
  "n71_answer": "Dermatologist (monthly)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "20 min",
  "n71_answer": "20 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "subscription_service",
  "question": "What am I subscribed to?",
  "gold_answer": "Dranthazon",
  "n71_answer": "Dranthazon, Netflix, Disney+, Hulu, Apple TV+, Peacock, HBO (considering canceling), Amazon Prime Video (planning to watch)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "Thyrelion (anime), a history documentary series, Xelvari Dawn (book)",
  "n71_answer": "The Crown, Stranger Things, Schitt's Creek, The Mandalorian, The Good Place, Grey's Anatomy, The Morning Show, The Office, Parks and Recreation",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "therapy (weekly)",
  "n71_answer": "therapy (weekly)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "Uncertain — previously '20 min', but residence_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "subscription_service",
  "question": "What am I subscribed to?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is cooking, plays running club, and is part of book club.",
  "n71_answer": "Cooking, running three times a week, attending trivia night on Wednesdays, spending time with partner Sarah, enjoying social aspect of fitness, participating in running club, book club, Our Project, Animal Shelter, watching history documentary series, and exploring new activities or places with Sarah.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-007",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "backend migration",
  "n71_answer": "backend migration project",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "walking",
  "n71_answer": "walking to work",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "interim coverage plan",
  "n71_answer": "interim coverage plan",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Zefiro's Kitchen",
  "n71_answer": "Zefiro's Kitchen",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "vehicle",
  "question": "What car do I drive?",
  "gold_answer": "Xylorim Scooter",
  "n71_answer": "Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, meal kit (Kelvimeal), Dranthazon",
  "n71_answer": "Xelvoria subscriber, Kelvimeal",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "marketing campaign",
  "n71_answer": "marketing campaign",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "bus",
  "n71_answer": "bus",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "Uncertain — previously 'interim coverage plan', but employer changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Uncertain — previously 'Zefiro's Kitchen', but work_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "vehicle",
  "question": "What car do I drive?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is cooking, plays cycling, and is part of board game group.",
  "n71_answer": "- Added new novels to collection  \n- Organized bookshelf  \n- Got a new area rug  \n- Replaced light bulb in table lamp  \n- Binge-watching TV shows  \n- Spending a lot of time on couch",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-008",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "low-sodium",
  "n71_answer": "low-sodium",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Zorathel Peak Gym",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "pilates 3x/week",
  "n71_answer": "pilates 3x/week",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "tutoring session (weekly)",
  "n71_answer": "weekly tutoring session",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Quorvel EV, Vorantel SUV",
  "n71_answer": "Zyvanta Sedan, Quorvel EV, Vorantel SUV",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "no alcohol",
  "n71_answer": "low-sodium, no alcohol",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Crysthene Pool",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "Uncertain — previously 'pilates 3x/week', but health_condition changed",
  "n71_answer": "yoga 2x/week",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is cooking, plays running club, and is part of book club.",
  "n71_answer": "Cooking",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-009",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "mountain ski trip",
  "n71_answer": "Mountain ski trip",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "with roommate",
  "n71_answer": "roommate",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "brunch with college friends monthly",
  "n71_answer": "brunch with college friends monthly",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "partner",
  "question": "List all partners I've had over time, from earliest to latest.",
  "gold_answer": "Alex, Jordan, Taylor",
  "n71_answer": "I don't have that information.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "Zynthera Bay beach trip (1 week)",
  "n71_answer": "Zynthera Bay beach trip (1 week)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "Uncertain — previously 'with roommate', but relationship_status changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "brunch with college friends monthly",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is video games, plays pickleball, and is part of board game group.",
  "n71_answer": "Listen to true crime podcasts, play video games, play pickleball, attend monthly brunch with college friends.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-010",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dermatologist (monthly)",
  "n71_answer": "Dermatologist (monthly)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "none",
  "n71_answer": "Swimming 2x/week",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Korathril",
  "n71_answer": "Korathril",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Zefiro's Kitchen",
  "n71_answer": "Zefiro's Kitchen",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, Daily Journal subscription, Pyrethon Plus",
  "n71_answer": "Daily Journal subscription -> Pyrethon Plus",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "therapy (weekly)",
  "n71_answer": "therapy (weekly)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "none",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Uncertain — previously 'Korathril', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Zefiro's Kitchen",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is gardening, plays running club, and is part of wine tasting club.",
  "n71_answer": "Gardening, swimming 2x/week, walking daily, attending trivia night Wednesdays, participating in running club, participating in wine tasting club, reading The Quorath Codex (book)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-011",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "alone",
  "n71_answer": "Alone",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "single-family house",
  "n71_answer": "single-family house",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "physical therapy (2x/week)",
  "n71_answer": "physical therapy 2 times a week",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Dranthis Pickup, Vorantel SUV",
  "n71_answer": "Zyvanta Sedan, Dranthis Pickup, Vorantel SUV",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "with partner",
  "n71_answer": "partner",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "Uncertain — previously 'single-family house', but relationship_status changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is gardening, plays rock climbing, and is part of volunteer group.",
  "n71_answer": "gardening, rock climbing, pickup basketball on Saturdays",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-012",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "shift work (night)",
  "n71_answer": "shift work (night)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "40 min",
  "n71_answer": "40 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "infrastructure upgrade",
  "n71_answer": "infrastructure upgrade project",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "10pm-6am",
  "n71_answer": "10pm to 6am",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "The Quorath Codex (book), Thyrelion (anime), Xelvari Dawn (book)",
  "n71_answer": "The Quorath Codex, Thyrelion (anime), Xelvari Dawn",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "4-day work week",
  "n71_answer": "shift work (night)",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "40 min",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "Uncertain — previously 'infrastructure upgrade', but employer changed",
  "n71_answer": "infrastructure upgrade project",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Uncertain — previously 'Pyravar CrossFit', but work_location changed",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "10pm to 6am",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is photography, plays golf, and is part of book club.",
  "n71_answer": "Photography, golf, book club, trivia night on Wednesdays, solo hike this weekend near the Bay Area",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-013",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dentist (every 6 months)",
  "n71_answer": "Dentist (every 6 months)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Velnorath",
  "n71_answer": "Velnorath",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "dental + vision bundle",
  "n71_answer": "dental and vision bundle",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Daily Journal subscription, Dranthazon, Pyrethon Plus",
  "n71_answer": "Daily Journal subscription",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dermatologist (monthly)",
  "n71_answer": "dermatologist (monthly)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Kaelstrom Fitness",
  "n71_answer": "Kaelstrom Fitness",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Uncertain — previously 'Velnorath', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is guitar, plays basketball, and is part of running club.",
  "n71_answer": "guitar, basketball, watching history documentary series, game night with neighbors",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-014",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "with parents",
  "n71_answer": "parents",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "sibling's wedding",
  "n71_answer": "Sibling's wedding",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "dairy-free",
  "n71_answer": "dairy-free",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "partner",
  "question": "List all partners I've had over time, from earliest to latest.",
  "gold_answer": "Jessica, Mike, James",
  "n71_answer": "No information on your partners over time is available.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "with sibling",
  "n71_answer": "sibling",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "Uncertain — previously 'sibling's wedding', but relationship_status changed",
  "n71_answer": "Sibling's wedding",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "dairy-free",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is photography, plays running club, and is part of hiking club.",
  "n71_answer": "Photography, antique collection, pickup basketball on Saturdays, hiking (as part of hiking club), running (as part of running club)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-015",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "cross-country road trip (2 weeks)",
  "n71_answer": "cross-country road trip (2 weeks)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "paying off student loans",
  "n71_answer": "paying_off student loans",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "health_condition",
  "question": "What health condition do I have?",
  "gold_answer": "lactose intolerance",
  "n71_answer": "lactose intolerance",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Therwyn Compact, Dranthis Pickup, Vorantel SUV",
  "n71_answer": "Therwyn Compact, Dranthis Pickup, Vorantel SUV, Subaru Outback",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "Zynthera Bay beach trip (1 week)",
  "n71_answer": "Pacific Coast Highway (PCH) road trip with friends",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "Uncertain — previously 'paying off student loans', but relationship_status changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "health_condition",
  "question": "What health condition do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is pottery, plays golf, and is part of board game group.",
  "n71_answer": "pottery, golf, hiking, game night with neighbors, board game group membership, experimenting with different seasonings and spices, watching The Veldrakon Spiral (TV show)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-016",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "physical therapy (2x/week)",
  "n71_answer": "physical therapy (2x/week)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Thandrel YMCA",
  "n71_answer": "Thandrel YMCA",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "health_condition",
  "question": "What health condition do I have?",
  "gold_answer": "high blood pressure",
  "n71_answer": "high blood pressure",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Therwyn Compact, Vorantel SUV",
  "n71_answer": "Zyvanta Sedan, Therwyn Compact, Vorantel SUV",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "chiropractor (biweekly)",
  "n71_answer": "chiropractor (biweekly)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Uncertain — previously 'Thandrel YMCA', but residence_location changed",
  "n71_answer": "Thandrel YMCA",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "health_condition",
  "question": "What health condition do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is photography, plays running club, and is part of wine tasting club.",
  "n71_answer": "Watercolor painting, photography (landscape photography), brunch with college friends monthly, running club, wine tasting club.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-017",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "walking",
  "n71_answer": "walking to work",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "Spanish",
  "n71_answer": "Spanish",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "stand-up desk",
  "n71_answer": "stand-up desk",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Therwyn Compact, Dranthis Pickup",
  "n71_answer": "Zyvanta Sedan, Therwyn Compact, Dranthis Pickup",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "commuter rail",
  "n71_answer": "Commuter rail",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "Uncertain — previously 'Spanish', but school changed",
  "n71_answer": "part-time JD at Belvona Harbor Law",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Stand-up desk",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is pottery, plays volleyball, and is part of running club.",
  "n71_answer": "Pottery",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-018",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "maxing out retirement fund",
  "n71_answer": "maxing out retirement fund",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "single-family house",
  "n71_answer": "single-family house",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "pickup basketball Saturdays",
  "n71_answer": "pickup basketball on Saturdays",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "The Quorath Codex (book), a daily news podcast, Thyrelion (anime)",
  "n71_answer": "The Quorath Codex -> daily news podcast -> Thyrelion (anime)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "saving for wedding",
  "n71_answer": "saving for wedding",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "Uncertain — previously 'single-family house', but relationship_status changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "pickup basketball on Saturdays",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "travel_plan,upcoming_event,family_event",
  "question": "What do I have coming up? List everything you know.",
  "gold_answer": "The user is planning hiking trip out west, has college reunion coming up, and has Thanksgiving gathering.",
  "n71_answer": "Thanksgiving gathering, college reunion, brunch and hike with Emily",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-019",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "20 min",
  "n71_answer": "20 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dentist (every 6 months)",
  "n71_answer": "dentist (every 6 months)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "team lead",
  "n71_answer": "team lead",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Therwyn Compact, Xylorim Scooter",
  "n71_answer": "Zyvanta Sedan, Xylorim Scooter, Therwyn Compact",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "30 min",
  "n71_answer": "around 30 min",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "Uncertain — previously 'dentist (every 6 months)', but residence_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "None",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays golf, and is part of running club.",
  "n71_answer": "woodworking, playing golf, jewelry-making, walking goats around the neighborhood every morning",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-020",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "flexible hours",
  "n71_answer": "flexible hours",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "marketplace health plan",
  "n71_answer": "marketplace health plan",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Kinthar Grill",
  "n71_answer": "Kinthar Grill",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "subscription_service",
  "question": "What am I subscribed to?",
  "gold_answer": "meal kit (Kelvimeal)",
  "n71_answer": "Kelvimeal",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Therwyn Compact, Vorantel SUV, Xylorim Scooter",
  "n71_answer": "Therwyn Compact, Vorantel SUV, Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "hybrid (3 office + 2 remote)",
  "n71_answer": "hybrid (3 office + 2 remote)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Velthari Studio",
  "n71_answer": "Velthari Studio",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "Uncertain — previously 'marketplace health plan', but employer changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Uncertain — previously 'Kinthar Grill', but work_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "subscription_service",
  "question": "What am I subscribed to?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "commute_method,commute_duration,work_location",
  "question": "Tell me everything you know about my commute.",
  "gold_answer": "The user commutes by subway for 50 min to Belvona Harbor.",
  "n71_answer": "Commute duration: 50 minutes  \nCommute method: subway",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-021",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "saving for kids' education fund",
  "n71_answer": "saving for kids' education fund",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "9-to-5 fixed",
  "n71_answer": "9-to-5 fixed",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "residence_location",
  "question": "Where do I live?",
  "gold_answer": "Belvona Harbor",
  "n71_answer": "Belvona Harbor",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "partner",
  "question": "List all partners I've had over time, from earliest to latest.",
  "gold_answer": "James, Alex, Taylor",
  "n71_answer": "James, Alex, Taylor",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "saving for wedding",
  "n71_answer": "saving for kids' education fund",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "Uncertain — previously '9-to-5 fixed', but school changed",
  "n71_answer": "9-to-5 fixed",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "residence_location",
  "question": "Where do I live?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Belvona Harbor",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays tennis, and is part of wine tasting club.",
  "n71_answer": "Woodworking, tennis, attending stand-up comedy open mics, participating in a wine tasting club, and socializing in a potluck dinner group.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-022",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "midnight-8am",
  "n71_answer": "midnight-8am",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Kaelstrom Fitness",
  "n71_answer": "Kaelstrom Fitness",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dentist (every 6 months)",
  "n71_answer": "dentist (every 6 months)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "employer",
  "question": "Where do I work?",
  "gold_answer": "Caeltura Systems",
  "n71_answer": "Caeltura Systems",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Quorvel EV, Xylorim Scooter",
  "n71_answer": "Zyvanta Sedan, Quorvel EV, Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "irregular",
  "n71_answer": "irregular",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "home gym",
  "n71_answer": "home gym",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "Uncertain — previously 'dentist (every 6 months)', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "employer",
  "question": "Where do I work?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is video games, plays basketball, and is part of book club.",
  "n71_answer": "Play video games, basketball, participate in a book club, pickup basketball on Saturdays, walking daily, yoga 2x/week",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-023",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "condo",
  "n71_answer": "condo",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "family reunion",
  "n71_answer": "family reunion",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "Zynthera Bay beach trip (1 week)",
  "n71_answer": "Zynthera Bay Beach Trip",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "partner",
  "question": "List all partners I've had over time, from earliest to latest.",
  "gold_answer": "Sarah, Jessica, Taylor",
  "n71_answer": "Sarah, Taylor",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "townhouse",
  "n71_answer": "townhouse",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "Uncertain — previously 'family reunion', but relationship_status changed",
  "n71_answer": "I don't have that information.",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "None",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is video games, plays basketball, and is part of hiking club.",
  "n71_answer": "Playing video games, basketball, hiking",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-024",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "marketing lead",
  "n71_answer": "marketing lead",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "walking",
  "n71_answer": "walking to work",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "hybrid (3 office + 2 remote)",
  "n71_answer": "hybrid (3 office + 2 remote)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Velmara Taqueria",
  "n71_answer": "Velmara Taqueria",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "relationship_status",
  "question": "What's my relationship status?",
  "gold_answer": "in a relationship",
  "n71_answer": "in a relationship",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, meal kit (Kelvimeal), Pyrethon Plus",
  "n71_answer": "Pyrethon Plus, Kelvimeal",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "staff engineer",
  "n71_answer": "staff engineer",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "bus",
  "n71_answer": "walking to work",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "Uncertain — previously 'hybrid (3 office + 2 remote)', but employer changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Uncertain — previously 'Velmara Taqueria', but work_location changed",
  "n71_answer": "Velmara Taqueria",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "relationship_status",
  "question": "What's my relationship status?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none — explicitly removed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays soccer, and is part of book club.",
  "n71_answer": "woodworking, soccer, musicals, plays, vinyl record collecting",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-025",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "bike",
  "n71_answer": "By bike",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Thandrel YMCA",
  "n71_answer": "Thandrel YMCA",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "tutoring session (weekly)",
  "n71_answer": "weekly tutoring session",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Thyranote, meal kit (Kelvimeal), Pyrethon Plus",
  "n71_answer": "Thyranote, Pyrethon Plus, Kelvimeal",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "subway",
  "n71_answer": "Subway",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Uncertain — previously 'Thandrel YMCA', but residence_location changed",
  "n71_answer": "Thandrel YMCA",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is guitar, plays pickleball, and is part of cycling club.",
  "n71_answer": "Engage in activities that help unwind, pursue interests, play guitar, and participate in trivia night on Wednesdays.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-026",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "10 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "physical therapy (2x/week)",
  "n71_answer": "physical therapy (2x/week)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "diet_preference",
  "question": "What's my diet?",
  "gold_answer": "keto",
  "n71_answer": "keto",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, Thyranote, Zorvista Premium",
  "n71_answer": "Xelvoria, Zorvista Premium",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "15 min",
  "n71_answer": "15 min",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "Uncertain — previously 'physical therapy (2x/week)', but residence_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "diet_preference",
  "question": "What's my diet?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "keto",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is photography, plays golf, and is part of hiking club.",
  "n71_answer": "photography, golf, reading Zadie Smith's short stories in The New Yorker, reading during daily commute, watching Whispers of Zarinthea (TV)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-027",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "staff engineer",
  "n71_answer": "staff engineer",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "10 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "interim coverage plan",
  "n71_answer": "interim coverage plan",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Velmara Taqueria",
  "n71_answer": "Velmara Taqueria",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "walking daily",
  "n71_answer": "walking daily",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Zorvista Premium, Dranthazon, Pyrethon Plus",
  "n71_answer": "Pyrethon Plus",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "designer",
  "n71_answer": "designer",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "20 min",
  "n71_answer": "20 min",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "Uncertain — previously 'interim coverage plan', but employer changed",
  "n71_answer": "interim coverage plan",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Uncertain — previously 'Velmara Taqueria', but work_location changed",
  "n71_answer": "Velmara Taqueria",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none — explicitly removed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is gardening, plays basketball, and is part of hiking club.",
  "n71_answer": "gardening, basketball",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-028",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "10-to-7 fixed",
  "n71_answer": "10-to-7 fixed",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "maxing out retirement fund",
  "n71_answer": "maxing out retirement fund",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "11pm-7am",
  "n71_answer": "11pm to 7am",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "Whispers of Zarinthea (TV), Thyrelion (anime), Xelvari Dawn (book)",
  "n71_answer": "Whispers of Zarinthea (TV), Thyrelion (anime)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "hybrid (3 office + 2 remote)",
  "n71_answer": "hybrid (3 office + 2 remote)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "Uncertain — previously 'maxing out retirement fund', but school changed",
  "n71_answer": "engagement ring",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "11pm to 7am",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is video games, plays volleyball, and is part of wine tasting club.",
  "n71_answer": "video games, volleyball",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-029",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "shift work (day)",
  "n71_answer": "shift work (day)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "10 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "employer + supplemental",
  "n71_answer": "employer + supplemental",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "remote (no commute)",
  "n71_answer": "Remote",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "paying off student loans",
  "n71_answer": "paying off student loans",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "The Quorath Codex (book), Thyrelion (anime), Xelvari Dawn (book)",
  "n71_answer": "Thyrelion (anime), Xelvari Dawn (book), The Quorath Codex (book)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "work_schedule",
  "question": "What's my work schedule?",
  "gold_answer": "9-to-5 fixed",
  "n71_answer": "9-to-5 fixed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "10 min",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "Uncertain — previously 'employer + supplemental', but employer changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "Uncertain — previously 'remote (no commute)', but work_location changed",
  "n71_answer": "I don't have that information.",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "financial_goal",
  "question": "What's my current financial goal?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none — explicitly removed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is video games, plays pickleball, and is part of cycling club.",
  "n71_answer": "video games, vintage camera collection, vintage postcard collecting, vinyl record collecting, coin collecting, stamp collecting, antique pocket watch collecting, pickleball",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-030",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "tutoring session (weekly)",
  "n71_answer": "tutoring session (weekly)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "midnight-8am",
  "n71_answer": "midnight-8am",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "1.5 hours",
  "n71_answer": "1.5 hours",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "a true crime podcast, The Quorath Codex (book), Xelvari Dawn (book)",
  "n71_answer": "Xelvari Dawn",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "dentist (every 6 months)",
  "n71_answer": "dentist (every 6 months)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "home gym",
  "n71_answer": "home gym",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "Uncertain — previously 'midnight-8am', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "I don't have that information.",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is cooking, plays cycling, and is part of volunteer group.",
  "n71_answer": "cooking, cycling, game night with neighbors, collecting Star Wars items, collecting antique jewelry, collecting coins, volunteering",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-031",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "baby shower",
  "n71_answer": "Baby shower",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "with roommate",
  "n71_answer": "roommate",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "new laptop",
  "n71_answer": "new laptop",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Thyranote, Zorvista Premium, meal kit (Kelvimeal)",
  "n71_answer": "Thyranote, Zorvista Premium, Kelvimeal",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "parent's birthday",
  "n71_answer": "Baby shower",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "Uncertain — previously 'with roommate', but relationship_status changed",
  "n71_answer": "Mike",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "New laptop",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is reading, plays basketball, and is part of running club.",
  "n71_answer": "Reading, playing basketball, running (running club)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-032",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "dental + vision bundle",
  "n71_answer": "dental + vision bundle",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Lorwen & Sage",
  "n71_answer": "Lorwen & Sage",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "infrastructure upgrade",
  "n71_answer": "Gourmet Gift Basket Project",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "about 10 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "diet_preference",
  "question": "What's my diet?",
  "gold_answer": "vegetarian",
  "n71_answer": "vegetarian diet",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "The Veldrakon Spiral (TV), a daily news podcast, Thyrelion (anime)",
  "n71_answer": "The Veldrakon Spiral (TV), Thyrelion (anime)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "employer-provided health plan",
  "n71_answer": "employer-provided health plan",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Velmara Taqueria",
  "n71_answer": "Velmara Taqueria",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "Uncertain — previously 'infrastructure upgrade', but employer changed",
  "n71_answer": "infrastructure upgrade project",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "Uncertain — previously '10 min', but work_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "diet_preference",
  "question": "What's my diet?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "vegetarian diet",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is pottery, plays pickleball, and is part of wine tasting club.",
  "n71_answer": "BBQ grilling, making homemade strawberry jam, pottery, pickleball, pickup basketball on Saturdays, wine tasting club.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-033",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "running 3x/week",
  "n71_answer": "running 3x/week",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Kaelstrom Fitness",
  "n71_answer": "Kaelstrom Fitness",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Korathril",
  "n71_answer": "Korathril",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "work_location",
  "question": "Where is my office?",
  "gold_answer": "remote",
  "n71_answer": "remote",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, Thyranote, Zorvista Premium",
  "n71_answer": "Xelvoria subscription, Thyranote subscription, Zorvista Premium",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "weight training 4x/week",
  "n71_answer": "weight training 4x/week",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Uncertain — previously 'Korathril', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "work_location",
  "question": "Where is my office?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays cycling, and is part of wine tasting club.",
  "n71_answer": "Woodworking, cycling, pickup basketball on Saturdays, wine tasting club.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-034",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "dormitory",
  "n71_answer": "dormitory",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "family reunion",
  "n71_answer": "Family reunion",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "employer",
  "question": "Where do I work?",
  "gold_answer": "Zypherix Labs",
  "n71_answer": "Zypherix Labs",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Zorvista Premium, meal kit (Kelvimeal), Pyrethon Plus",
  "n71_answer": "Zorvista Premium, Kelvimeal, Pyrethon Plus",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "two-bedroom apartment",
  "n71_answer": "two-bedroom apartment",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "Uncertain — previously 'family reunion', but relationship_status changed",
  "n71_answer": "family reunion",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "employer",
  "question": "Where do I work?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is reading, plays running club, and is part of volunteer group.",
  "n71_answer": "reading, tailgating, watching a history documentary series",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-035",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "30 min",
  "n71_answer": "30 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Zorathel Peak Gym",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "electric car",
  "n71_answer": "electric car",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "partner",
  "question": "List all partners I've had over time, from earliest to latest.",
  "gold_answer": "James, Jordan, Taylor",
  "n71_answer": "James, Jordan, Taylor",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "10 min",
  "n71_answer": "10 min",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Uncertain — previously 'Zorathel Peak Gym', but residence_location changed",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "planned_purchase",
  "question": "What big purchase am I planning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "electric car",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is drawing/painting, plays running club, and is part of wine tasting club.",
  "n71_answer": "drawing/painting, listening to vinyl records, collecting coins, cycling, game night with neighbors",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-036",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "child's school event",
  "n71_answer": "Child's school event",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "shared house",
  "n71_answer": "shared house",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "employer",
  "question": "Where do I work?",
  "gold_answer": "Korvath & Co.",
  "n71_answer": "Korvath & Co.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "a daily news podcast, Thyrelion (anime), a history documentary series",
  "n71_answer": "daily news podcast, Thyrelion (anime), history documentary series",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "none currently",
  "n71_answer": "Child's school event",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "Uncertain — previously 'shared house', but relationship_status changed",
  "n71_answer": "shared house",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "employer",
  "question": "Where do I work?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is gardening, plays volleyball, and is part of wine tasting club.",
  "n71_answer": "Gardening, volleyball, skiing, wine tasting club, potluck dinner group.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-037",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "cross-country road trip (2 weeks)",
  "n71_answer": "Cross-country road trip (2 weeks)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "sibling's wedding",
  "n71_answer": "Sibling's wedding",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "work_location",
  "question": "Where is my office?",
  "gold_answer": "Thornvale Crossing",
  "n71_answer": "Thornvale Crossing",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Therwyn Compact, Quorvel EV, Vorantel SUV",
  "n71_answer": "Therwyn Compact, Quorvel EV, Vorantel SUV",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "island getaway",
  "n71_answer": "Cross-country road trip (2 weeks)",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "Uncertain — previously 'sibling's wedding', but relationship_status changed",
  "n71_answer": "Sibling's wedding",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "work_location",
  "question": "Where is my office?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Thornvale Crossing",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is photography, plays running club, and is part of cycling club.",
  "n71_answer": "Photography, learning coding, web development, watching Whispers of Zarinthea (TV)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-038",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "Spanish",
  "n71_answer": "Spanish",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "bike",
  "n71_answer": "bike",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "no caffeine",
  "n71_answer": "no caffeine",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "media_consumption",
  "question": "List all media I've consumed over time, from earliest to latest.",
  "gold_answer": "Whispers of Zarinthea (TV), a true crime podcast, Xelvari Dawn (book)",
  "n71_answer": "Whispers of Zarinthea (TV), true crime podcast, Xelvari Dawn (book)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "skill_acquisition",
  "question": "What am I currently learning?",
  "gold_answer": "Japanese",
  "n71_answer": "Spanish",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "Uncertain — previously 'bike', but school changed",
  "n71_answer": "Bike",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "no caffeine",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is knitting, plays soccer, and is part of running club.",
  "n71_answer": "knitting, soccer",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-039",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "dairy-free",
  "n71_answer": "dairy-free",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "none",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Velnorath",
  "n71_answer": "Velnorath",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "bus",
  "n71_answer": "By bus",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Quorvel EV, Vorantel SUV",
  "n71_answer": "Zyvanta Sedan, Quorvel EV, Vorantel SUV",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "nut-free",
  "n71_answer": "nut-free",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Uncertain — previously 'Velnorath', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "You don't have a commute method currently; it has been explicitly removed.",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays volleyball, and is part of cycling club.",
  "n71_answer": "Woodworking, volleyball, watching Whispers of Zarinthea (TV), attending game night with neighbors.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-040",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Korathril",
  "n71_answer": "Korathril",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Zorathel Peak Gym",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "yoga 2x/week",
  "n71_answer": "yoga 2x/week",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "hiking trip out west",
  "n71_answer": "California Coast Road Trip",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Dranthis Pickup, Vorantel SUV, Xylorim Scooter",
  "n71_answer": "Dranthis Pickup, Vorantel SUV, Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Dyntheral",
  "n71_answer": "Dyntheral",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Velthari Studio",
  "n71_answer": "Velthari Studio",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "Uncertain — previously 'yoga 2x/week', but health_condition changed",
  "n71_answer": "yoga 2x/week",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "California Coast Road Trip",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is photography, plays pickleball, and is part of book club.",
  "n71_answer": "Photography, pickleball, yoga 2x/week, attending trivia night on Wednesdays, watching the anime Thyrelion, reading as a member of a book club, and enjoying outdoor activities such as hiking and scenic drives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-041",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "n71_answer": "In three words I can sum up everything I've learned about life: it goes on. You'll have bad times, but it will always wake you up to the stuff you weren't paying attention to.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "walking daily",
  "n71_answer": "walking daily",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Velthari Studio",
  "n71_answer": "Velthari Studio",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Velnorath",
  "n71_answer": "Velnorath",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "Zynthera Bay beach trip (1 week)",
  "n71_answer": "Zynthera Bay beach trip for 1 week",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Zyvanta Sedan, Therwyn Compact, Xylorim Scooter",
  "n71_answer": "Zyvanta Sedan, Therwyn Compact, Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "swimming 2x/week",
  "n71_answer": "swimming 2x/week",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "none",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "medication",
  "question": "What medication am I taking?",
  "gold_answer": "Uncertain — previously 'Velnorath', but health_condition changed",
  "n71_answer": "None",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Zynthera Bay beach trip",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is guitar, plays volleyball, and is part of running club.",
  "n71_answer": "Play guitar, play volleyball, practice volleyball as a setter.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-042",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "n71_answer": "Twenty years from now you will be more disappointed by the things that you didn't do than by the ones you did do. So throw off the bowlines, sail away from safe harbor, catch the trade winds in your sails. Explore, dream, discover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "no travel planned",
  "n71_answer": "No travel planned",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "baby shower",
  "n71_answer": "baby shower",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "chiropractor (biweekly)",
  "n71_answer": "chiropractor every two weeks",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, Thyranote, Zorvista Premium",
  "n71_answer": "Xelvoria subscription, Thyranote subscription, Zorvista Premium",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "island getaway",
  "n71_answer": "island getaway",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "Uncertain — previously 'baby shower', but relationship_status changed",
  "n71_answer": "baby shower",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is knitting, plays rock climbing, and is part of book club.",
  "n71_answer": "knitting, rock climbing, pickup basketball on Saturdays, participating in a small film festival, engaging with Xelvoria and Thyranote subscriptions, attending a book club, reading The Quorath Codex, shopping online more frequently",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-043",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "varies by shift",
  "n71_answer": "varies by shift",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Velthari Studio",
  "n71_answer": "Velthari Studio",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "walking daily",
  "n71_answer": "walking daily",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "game night with neighbors",
  "n71_answer": "game night with neighbors",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, Zorvista Premium, meal kit (Kelvimeal)",
  "n71_answer": "Xelvoria (2023-03-05) -> Zorvista Premium (2023-03-08)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "1am-9am",
  "n71_answer": "1am-9am",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Thandrel YMCA",
  "n71_answer": "Thandrel YMCA",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "exercise_routine",
  "question": "What's my exercise routine?",
  "gold_answer": "Uncertain — previously 'walking daily', but health_condition changed",
  "n71_answer": "yoga 2x/week",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "game night with neighbors",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is knitting, plays basketball, and is part of running club.",
  "n71_answer": "sourdough bread baking, knitting, basketball, hosting BBQ parties",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-044",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "mountain ski trip",
  "n71_answer": "Mountain ski trip",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "with partner and child",
  "n71_answer": "partner and child",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "no alcohol",
  "n71_answer": "no alcohol",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, Zorvista Premium, Daily Journal subscription",
  "n71_answer": "Daily Journal subscription, Zorvista Premium",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "travel_plan",
  "question": "What trip am I planning?",
  "gold_answer": "no travel planned",
  "n71_answer": "Mountain ski trip",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "living_arrangement",
  "question": "Who do I live with?",
  "gold_answer": "Uncertain — previously 'with partner and child', but relationship_status changed",
  "n71_answer": "James and child",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "no alcohol",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is cooking, plays volleyball, and is part of wine tasting club.",
  "n71_answer": "Cooking, volleyball, playing games, reading, books",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-045",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "n71_answer": "Most people never run far enough on their first wind to find out they've got a second. Give your dreams all you've got and you'll be amazed at the energy that comes out of you.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "dairy-free",
  "n71_answer": "dairy-free",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "11pm-5:30am (early riser)",
  "n71_answer": "11pm-5:30am (early riser)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "school",
  "question": "Where am I studying?",
  "gold_answer": "Quivira Springs Community College",
  "n71_answer": "Quivira Springs Community College",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, meal kit (Kelvimeal), Dranthazon",
  "n71_answer": "Kelvimeal, Dranthazon",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "nut-free",
  "n71_answer": "nut-free",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Crysthene Pool",
  "n71_answer": "Pyravar CrossFit",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "Uncertain — previously '11pm-5:30am (early riser)', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "school",
  "question": "Where am I studying?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Quivira Springs Community College",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is pottery, plays pickleball, and is part of book club.",
  "n71_answer": "Pottery, pickleball, participating in a potluck dinner group, and being part of a book club.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-046",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "designer",
  "n71_answer": "designer",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "1 hour",
  "n71_answer": "1 hour",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "work_location",
  "question": "Where is my office?",
  "gold_answer": "Tesswick Bay",
  "n71_answer": "Tesswick Bay",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "bike",
  "n71_answer": "By bike",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "pickup basketball Saturdays",
  "n71_answer": "pickup basketball group",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "vehicle",
  "question": "List all vehicles I've had over time, from earliest to latest.",
  "gold_answer": "Therwyn Compact, Vorantel SUV, Xylorim Scooter",
  "n71_answer": "Therwyn Compact, Vorantel SUV, Xylorim Scooter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "job_title",
  "question": "What is my job title?",
  "gold_answer": "product manager",
  "n71_answer": "product manager",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "1.5 hours",
  "n71_answer": "1.5 hours",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "work_location",
  "question": "Where is my office?",
  "gold_answer": "Uncertain — previously 'Tesswick Bay', but employer changed",
  "n71_answer": "Quivira Springs",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "commute_method",
  "question": "How do I get to work?",
  "gold_answer": "Uncertain — previously 'bike', but work_location changed",
  "n71_answer": "By Xylorim Scooter.",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "social_gathering",
  "question": "What's my regular social meetup?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "pickup basketball Saturdays",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is gardening, plays basketball, and is part of book club.",
  "n71_answer": "gardening, collecting Pokemon cards, collecting vintage video games, collecting action figures, collecting Funko Pops, basketball",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-047",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.",
  "n71_answer": "\"We are what we repeatedly do. Excellence, then, is not an act, but a habit. The whole is greater than the sum of its parts. We must become the change we wish to see in the world.\"",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "low-sugar",
  "n71_answer": "low-sugar",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Velthari Studio",
  "n71_answer": "Velthari Studio",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "midnight-8am",
  "n71_answer": "midnight-8am",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "physical therapy (2x/week)",
  "n71_answer": "physical therapy",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "partner",
  "question": "List all partners I've had over time, from earliest to latest.",
  "gold_answer": "Jessica, Alex, Chris",
  "n71_answer": "Jessica, Alex, Chris",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "dietary_restriction",
  "question": "What dietary restrictions do I have?",
  "gold_answer": "no processed food",
  "n71_answer": "no processed food",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Kaelstrom Fitness",
  "n71_answer": "Kaelstrom Fitness",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "sleep_pattern",
  "question": "What's my sleep schedule?",
  "gold_answer": "Uncertain — previously 'midnight-8am', but health_condition changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "regular_appointment",
  "question": "What regular appointments do I have?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays cycling, and is part of cycling club.",
  "n71_answer": "woodworking, cycling",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-048",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "employer + supplemental",
  "n71_answer": "employer + supplemental",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Velthari Studio",
  "n71_answer": "Velthari Studio",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "mobile app development",
  "n71_answer": "mobile app development",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Zefiro's Kitchen",
  "n71_answer": "Zefiro's Kitchen",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "anniversary dinner",
  "n71_answer": "Anniversary dinner",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "\"The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.\"",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Xelvoria, Dranthazon, Pyrethon Plus",
  "n71_answer": "Pyrethon Plus",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "insurance",
  "question": "What insurance do I have?",
  "gold_answer": "dental + vision bundle",
  "n71_answer": "dental + vision bundle",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Kaelstrom Fitness",
  "n71_answer": "Kaelstrom Fitness",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "work_project",
  "question": "What project am I working on?",
  "gold_answer": "Uncertain — previously 'mobile app development', but employer changed",
  "n71_answer": "mobile app development",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "favorite_restaurant",
  "question": "What's my favorite restaurant?",
  "gold_answer": "Uncertain — previously 'Zefiro's Kitchen', but work_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "family_event",
  "question": "What family event do I have coming up?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is guitar, plays golf, and is part of book club.",
  "n71_answer": "Play guitar, golf, fantasy football, watch Whispers of Zarinthea (TV), enjoy horror TV shows, comic book adaptations, video games, Moorish architecture, Gothic architecture, monthly brunch with college friends.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-049",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "n71_answer": "The only way to do great work is to love what you do. If you haven't found it yet, keep looking. Don't settle. As with all matters of the heart, you'll know when you find it.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "before",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Pyravar CrossFit",
  "n71_answer": "Pyravar CrossFit (current) or Zorathel Peak Gym (if residence location changes)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "before",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "50 min",
  "n71_answer": "50 min",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "before",
  "task": "Del",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "single-family house",
  "n71_answer": "single-family house",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "before",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "after",
  "task": "Tr",
  "entity": "subscription_service",
  "question": "List all subscription services I've had over time, from earliest to latest.",
  "gold_answer": "Thyranote, meal kit (Kelvimeal), Dranthazon",
  "n71_answer": "Kelvimeal, Dranthazon",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "after",
  "task": "Cas",
  "entity": "fitness_facility",
  "question": "Where do I work out?",
  "gold_answer": "Zorathel Peak Gym",
  "n71_answer": "Zorathel Peak Gym",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "after",
  "task": "Abs",
  "entity": "commute_duration",
  "question": "How long is my commute?",
  "gold_answer": "Uncertain — previously '50 min', but residence_location changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "after",
  "task": "Del",
  "entity": "housing_type",
  "question": "What type of home do I live in?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "after",
  "task": "Agg",
  "entity": "hobby,sports,club_membership",
  "question": "What do I do in my free time? List everything you know.",
  "gold_answer": "The user's hobby is woodworking, plays basketball, and is part of volunteer group.",
  "n71_answer": "woodworking, cooking, baking, basketball, photography",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "pl-050",
  "domain": "pl",
  "phase": "after",
  "task": "ER",
  "entity": "life_philosophy",
  "question": "Recite my life philosophy verbatim.",
  "gold_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "n71_answer": "Try not to become a man of success, but rather try to become a man of value. Look around at how people want to get more out of life than they put in. A man of value will give more than he receives.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Zelthra Pack",
  "n71_answer": "Zelthra Pack",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "zelthra pack --sourcemap",
  "n71_answer": "zelthra pack --sourcemap",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "single-app with /src and /lib",
  "n71_answer": "single-app with /src and /lib",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "tarvex verify --integration",
  "n71_answer": "tarvex verify --integration",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "documentation_url",
  "question": "Where are our docs?",
  "gold_answer": "github.com/aurora-team/docs",
  "n71_answer": "github.com/aurora-team/docs",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 April 11, 2025 May 23, 2025 June 20",
  "n71_answer": "2025 April 11, 2025 May 23, 2025 June 20",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Quorix Forge",
  "n71_answer": "Quorix Forge",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "quorix forge --target prod",
  "n71_answer": "quorix forge --target prod",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "Uncertain — previously 'single-app with /src and /lib', but framework changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "Uncertain — previously 'tarvex verify --integration', but test_framework changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "documentation_url",
  "question": "Where are our docs?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "github.com/aurora-team/docs",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "meeting_day,slack_channel,standup_format",
  "question": "How does our team coordinate? Tell me everything you know.",
  "gold_answer": "Team meets every Monday 11am, uses #aurora-dev for chat, and standups are async written updates in Slack.",
  "n71_answer": "The team coordinates through the #aurora-dev channel for development and project updates, holds team meetings every Monday at 11am, uses async written updates in Slack for standups, and follows a volunteer-based weekly signup for on-call rotation.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-001",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "orinthal-build.yml",
  "n71_answer": "orinthal-build.yml",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "deploy/Dockerfile.korinth",
  "n71_answer": "deploy/Dockerfile.korinth",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "stream.velturis.io/aurora",
  "n71_answer": "stream.velturis.io/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "orinthal publish --tier production",
  "n71_answer": "orinthal publish --tier production",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "default_branch",
  "question": "What's our default branch?",
  "gold_answer": "develop",
  "n71_answer": "develop",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 April 25, 2025 June 6, 2025 June 20",
  "n71_answer": "2025 April 25, 2025 June 6, 2025 June 20",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "korinth-deploy.yaml",
  "n71_answer": "orinthal-build.yml",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "deploy/Dockerfile.zephyra",
  "n71_answer": "deploy/Dockerfile.zephyra",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "Uncertain — previously 'stream.velturis.io/aurora', but deploy_target changed",
  "n71_answer": "stream.velturis.io/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "Uncertain — previously 'orinthal publish --tier production', but ci_config changed",
  "n71_answer": "orinthal publish --tier production",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "default_branch",
  "question": "What's our default branch?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "develop",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Synthari Elements v1.2, target 90% for core modules test coverage, and require lead approval for core/ for reviews.",
  "n71_answer": "lead approval for the core/ directory, test coverage target: 90% for core modules",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-002",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "drain.zephyra.io/proj-42",
  "n71_answer": "drain.zephyra.io/proj-42",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "thandrel ship --channel beta",
  "n71_answer": "thandrel ship --channel beta",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "sandbox.thandrel.io/aurora",
  "n71_answer": "sandbox.thandrel.io/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/zephyra/Dockerfile",
  "n71_answer": "infra/zephyra/Dockerfile",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "team_lead",
  "question": "Who's the team lead?",
  "gold_answer": "Sarah Chen",
  "n71_answer": "Sarah Chen",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Quorix Hub, Synthari Modules",
  "n71_answer": "Karnex Pack, Quorix Hub, Synthari Modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "sink.orinthal.dev/aurora",
  "n71_answer": "drain.zephyra.io/proj-42",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "narvex deploy --env prod",
  "n71_answer": "thandrel ship --channel beta",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "Uncertain — previously 'sandbox.thandrel.io/aurora', but deploy_target changed",
  "n71_answer": "sandbox.thandrel.io/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/zephyra/Dockerfile', but docker_image changed",
  "n71_answer": "infra/zephyra/Dockerfile",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "team_lead",
  "question": "Who's the team lead?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "I don't have that information.",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use trunk-based with feature flags branching on production, releasing monthly first Monday.",
  "n71_answer": "- The team's default branch is production.  \n- The release schedule is monthly on the first Monday.  \n- Releases are managed from the production branch with best practices such as thorough testing before merging, using a staging environment for final checks, maintaining a clear versioning system, using tags in version control to mark release points, and communicating releases with the team.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-003",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "David Lee",
  "n71_answer": "David Lee",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "Taehyun Kim",
  "n71_answer": "Taehyun Kim",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "test_coverage_target",
  "question": "What's our test coverage target?",
  "gold_answer": "90% for core modules",
  "n71_answer": "90% for core modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 March 28, 2025 April 11, 2025 June 6",
  "n71_answer": "2025 March 28, April 11, 2025, 2025 June 6",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Seoyun Choi",
  "n71_answer": "Seoyun Choi",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "Uncertain — previously 'Taehyun Kim', but team_lead changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "test_coverage_target",
  "question": "What's our test coverage target?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none — explicitly removed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is paired oncall (primary + secondary), alerts go to #aurora-alerts, and runbook is at github.com/aurora-team/runbooks.",
  "n71_answer": "On-call rotation type: paired oncall  \nOn-call structure: primary and a secondary oncall for support",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-004",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "monitor.narvex.io/proj-aurora",
  "n71_answer": "monitor.narvex.io/proj-aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "narvex deploy --env prod",
  "n71_answer": "narvex deploy --env prod",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "dev.orinthal.dev/aurora",
  "n71_answer": "dev.orinthal.dev/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/zephyra/Dockerfile",
  "n71_answer": "infra/zephyra/Dockerfile",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "meeting_day",
  "question": "When's our team meeting?",
  "gold_answer": "every Thursday 2pm",
  "n71_answer": "Every Thursday at 2pm",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 April 25, 2025 May 23, 2025 June 6",
  "n71_answer": "2025 April 25, 2025 May 23, 2025 June 6",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "observe.velturis.io/aurora",
  "n71_answer": "observe.velturis.io/aurora",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "orinthal publish --tier sandbox",
  "n71_answer": "narvex deploy --env prod",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "Uncertain — previously 'dev.orinthal.dev/aurora', but deploy_target changed",
  "n71_answer": "dev.orinthal.dev/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/zephyra/Dockerfile', but docker_image changed",
  "n71_answer": "infra/zephyra/Dockerfile",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "meeting_day",
  "question": "When's our team meeting?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at confluence.aurora.dev/aurora, onboarding guide at wiki.aurora.dev/getting-started, and changelog at GitHub Releases page.",
  "n71_answer": "confluence.aurora.dev/aurora",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-005",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "James Lee",
  "n71_answer": "James Lee",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Seoyun Choi",
  "n71_answer": "Seoyun Choi",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "alert_channel",
  "question": "What's our alert channel?",
  "gold_answer": "#aurora-alerts",
  "n71_answer": "#aurora-alerts",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Cryptara Vault, Selvex Secrets, Pyronos Crypt",
  "n71_answer": "Cryptara Vault, Selvex Secrets, Pyronos Crypt",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "Helen Chen",
  "n71_answer": "James Lee",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Uncertain — previously 'Seoyun Choi', but team_lead changed",
  "n71_answer": "Seoyun Choi",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "alert_channel",
  "question": "What's our alert channel?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "#aurora-alerts",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "meeting_day,slack_channel,standup_format",
  "question": "How does our team coordinate? Tell me everything you know.",
  "gold_answer": "Team meets every Friday 9:30am, uses #aurora-dev for chat, and standups are async written updates in Slack.",
  "n71_answer": "Our team coordinates using async written updates in Slack as the standup format and communicates through the #aurora-dev channel. Meetings are scheduled every Friday at 9:30am.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-006",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "orm_library",
  "question": "What ORM do we use?",
  "gold_answer": "Crysthene ORM",
  "n71_answer": "Crysthene ORM",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Crysthene.define('Model', {fields})",
  "n71_answer": "Crysthene.define('Model', {fields})",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "zyndra://base-main:9042/aurora",
  "n71_answer": "zyndra://base-main:9042/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "stream.velturis.io/aurora",
  "n71_answer": "stream.velturis.io/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Cryptara Vault, Verisigil Lock, Sigilix Manager",
  "n71_answer": "Cryptara Vault -> Verisigil Lock -> Sigilix Manager",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "orm_library",
  "question": "What ORM do we use?",
  "gold_answer": "Thalnex Query",
  "n71_answer": "Thalnex Query",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Thalnex.document('Model', {fields})",
  "n71_answer": "Thalnex.document('Model', {fields})",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "Uncertain — previously 'zyndra://base-main:9042/aurora', but database changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is weekly Mon-Sun rotation, alerts go to #aurora-oncall-alerts, and runbook is at runbooks.aurora.internal/playbooks.",
  "n71_answer": "weekly Mon-Sun rotation",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-007",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Verithos-JWT-v2",
  "n71_answer": "Verithos-JWT-v2",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "verithos-oauth-middleware v2.1",
  "n71_answer": "verithos-oauth-middleware v2.1",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "login_endpoint",
  "question": "What's the login endpoint?",
  "gold_answer": "/auth/kryptal/signin",
  "n71_answer": "/auth/kryptal/signin",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "watch.korinth.dev/aurora",
  "n71_answer": "watch.korinth.dev/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "I don't have that information.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 April 25, 2025 May 9, 2025 June 20",
  "n71_answer": "2025 April 25, 2025 May 9, 2025 June 20",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Oathkeeper-mTLS-cert",
  "n71_answer": "Verithos-JWT-v2",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "phalanx-key-guard v1.3",
  "n71_answer": "phalanx-key-guard v1.3",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "login_endpoint",
  "question": "What's the login endpoint?",
  "gold_answer": "Uncertain — previously '/auth/kryptal/signin', but auth_provider changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use GitFlow with release branches branching on release, releasing sprint-end release every 2 weeks.",
  "n71_answer": "- The team uses GitFlow with release branches branching.  \n- The default branch is named \"release,\" where all stable code lives before production.  \n- The team releases a sprint-end release every 2 weeks, maintaining momentum and staying on track with goals.  \n- Sprint deadlines have been updated over time, with the latest deadline set for 2025 June 20.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-008",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "korinth-deploy.yaml",
  "n71_answer": "korinth-deploy.yaml",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "korinth release --canary",
  "n71_answer": "korinth release --canary",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "velturis-cr/aurora:main",
  "n71_answer": "velturis-cr/aurora:main",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "deploy/Dockerfile.velturis",
  "n71_answer": "deploy/Dockerfile.velturis",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "changelog_location",
  "question": "Where's the changelog?",
  "gold_answer": "CHANGELOG.md in repo root",
  "n71_answer": "CHANGELOG.md in the repo root",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Pyronis Bundle, Quorix Hub",
  "n71_answer": "Karnex Pack, Pyronis Bundle, Quorix Hub",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "zephyra-pipeline.json",
  "n71_answer": "zephyra-pipeline.json",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "zephyra push --target preview",
  "n71_answer": "korinth release --canary",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "Uncertain — previously 'velturis-cr/aurora:main', but deploy_target changed",
  "n71_answer": "velturis-cr/aurora:main",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'deploy/Dockerfile.velturis', but docker_image changed",
  "n71_answer": "deploy/Dockerfile.velturis",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "changelog_location",
  "question": "Where's the changelog?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Quenthar Design v1, target no strict target, PR-reviewed test coverage, and require 2 approvals + security review for auth/ for reviews.",
  "n71_answer": "2 approvals + security review for auth/",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-009",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "zephyra-pipeline.json",
  "n71_answer": "zephyra-pipeline.json",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "zephyra push --target preview",
  "n71_answer": "zephyra push --target preview",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "zephyra-hub/aurora:prod",
  "n71_answer": "zephyra-hub/aurora:prod",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "deploy/Dockerfile.zephyra",
  "n71_answer": "deploy/Dockerfile.zephyra",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "logs.narvex.io/aurora/drain",
  "n71_answer": "logs.narvex.io/aurora/drain",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Verisigil Lock, Kerberos Trove, Orinthal Sealed",
  "n71_answer": "Verisigil Lock, Kerberos Trove, Orinthal Sealed",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "thandrel-flow.toml",
  "n71_answer": "zephyra-pipeline.json",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "thandrel ship --channel beta",
  "n71_answer": "zephyra push --target preview",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "Uncertain — previously 'zephyra-hub/aurora:prod', but deploy_target changed",
  "n71_answer": "zephyra-hub/aurora:prod",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'deploy/Dockerfile.zephyra', but docker_image changed",
  "n71_answer": "deploy/Dockerfile.zephyra",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "logs.narvex.io/aurora/drain",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use GitHub Flow with PR reviews branching on main, releasing monthly first Monday.",
  "n71_answer": "Our Project — release_cadence: monthly on the first Monday (current)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-010",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "dev_server_command",
  "question": "How do I run the dev server?",
  "gold_answer": "korvel dev --open",
  "n71_answer": "moxira serve --hot",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "karnex test --watch",
  "n71_answer": "quorix spec --run smoke",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Karnex Bundler",
  "n71_answer": "Vagrant",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "karnex build --analyze",
  "n71_answer": "karnex build --analyze",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "Session Cookie + CSRF",
  "n71_answer": "Session Cookie + CSRF",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Dranith Stack, Synthari Modules",
  "n71_answer": "Karnex Pack -> Dranith Stack -> Synthari Modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "dev_server_command",
  "question": "How do I run the dev server?",
  "gold_answer": "moxira serve --hot",
  "n71_answer": "korvel dev --open",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "quorix spec --run smoke",
  "n71_answer": "karnex test --watch",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Uncertain — previously 'Karnex Bundler', but framework changed",
  "n71_answer": "Karnex Bundler",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "Uncertain — previously 'karnex build --analyze', but build_tool changed",
  "n71_answer": "karnex build --analyze",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Session Cookie + CSRF",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Quenthar Design v1, target no strict target, PR-reviewed test coverage, and require lead approval for core/ for reviews.",
  "n71_answer": "- Code review policy: lead approval for the core/ directory  \n- Test coverage target: no strict target; PR-reviewed",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-011",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "orm_library",
  "question": "What ORM do we use?",
  "gold_answer": "Pyravar Mapper",
  "n71_answer": "Pyravar Mapper",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Pyravar.entity('Model', attrs)",
  "n71_answer": "Pyravar.entity('Model', attrs)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "migration_tool",
  "question": "What migration tool do we use?",
  "gold_answer": "Zyndra Alter",
  "n71_answer": "Zyndra Alter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Cerberix-SAML-assertion",
  "n71_answer": "Cerberix-SAML-assertion",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Cryptara Vault, Selvex Secrets, Pyronos Crypt",
  "n71_answer": "Cryptara Vault -> Selvex Secrets -> Pyronos Crypt",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "orm_library",
  "question": "What ORM do we use?",
  "gold_answer": "Vorathis ODM",
  "n71_answer": "Vorathis ODM",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Vorathis.collection('Model', spec)",
  "n71_answer": "none",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "migration_tool",
  "question": "What migration tool do we use?",
  "gold_answer": "Uncertain — previously 'Zyndra Alter', but database changed",
  "n71_answer": "Zyndra Alter",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at aurora-docs.readthedocs.io, onboarding guide at notion.so/aurora-team/new-hire, and changelog at GitHub Releases page.",
  "n71_answer": "aurora-docs.readthedocs.io",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-012",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "15 minutes",
  "n71_answer": "15 minutes",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "verithos-oauth-middleware v2.1",
  "n71_answer": "verithos-oauth-middleware v2.1",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "SAML 2.0 SSO",
  "n71_answer": "SAML 2.0 SSO",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "package_manager",
  "question": "What package manager do we use?",
  "gold_answer": "Dranith Stack",
  "n71_answer": "Dranith Stack",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 May 23, 2025 June 20, 2025 July 4",
  "n71_answer": "2025 May 23, 2025 June 20, 2025 July 4",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "4 hours",
  "n71_answer": "4 hours",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "oathkeeper-cert-check v3.2",
  "n71_answer": "oathkeeper-cert-check v3.2",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "Uncertain — previously 'SAML 2.0 SSO', but auth_provider changed",
  "n71_answer": "mTLS Client Cert",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "package_manager",
  "question": "What package manager do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is paired oncall (primary + secondary), alerts go to #aurora-oncall-alerts, and runbook is at wiki.aurora.dev/incident-response.",
  "n71_answer": "The team's on-call rotation is paired oncall (primary + secondary). This setup helps ensure that there's always someone available to tackle issues. Production alerts go to #aurora-oncall-alerts to streamline incident response. The on-call rotation type is paired oncall (primary + secondary).",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-013",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Kevin Park",
  "n71_answer": "Kevin Park",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "approval_authority",
  "question": "Who has approval authority?",
  "gold_answer": "CTO Soojung Lim",
  "n71_answer": "CTO Soojung Lim",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "thandrel ship --channel beta",
  "n71_answer": "thandrel ship --channel beta",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Cryptara Vault, Selvex Secrets, Kerberos Trove",
  "n71_answer": "Cryptara Vault -> Selvex Secrets -> Kerberos Trove",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Rachel Yoo",
  "n71_answer": "Rachel Yoo",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "approval_authority",
  "question": "Who has approval authority?",
  "gold_answer": "Uncertain — previously 'CTO Soojung Lim', but team_lead changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none — explicitly removed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Dravex Blocks v2.4, target no strict target, PR-reviewed test coverage, and require 2 approvals required for reviews.",
  "n71_answer": "- Code review policy: 2 approvals for a pull request to be merged  \n- Review process: everything is PR-reviewed  \n- Test coverage target: no strict target",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-014",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "status.orinthal.dev/aurora",
  "n71_answer": "status.orinthal.dev/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "narvex deploy --env prod",
  "n71_answer": "narvex deploy --env prod",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "preview.zephyra.io/aurora",
  "n71_answer": "preview.zephyra.io/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "deploy/Dockerfile.orinthal",
  "n71_answer": "deploy/Dockerfile.orinthal",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Pyronis Build",
  "n71_answer": "Pyronis Build",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "I don't have that information.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Tarvex Mod, Zelthra Lib, Dranith Stack",
  "n71_answer": "Tarvex Mod, Zelthra Lib, Dranith Stack",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "metrics.thandrel.io/aurora-prod",
  "n71_answer": "status.orinthal.dev/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "thandrel ship --channel beta",
  "n71_answer": "narvex deploy --env prod",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "Uncertain — previously 'preview.zephyra.io/aurora', but deploy_target changed",
  "n71_answer": "preview.zephyra.io/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'deploy/Dockerfile.orinthal', but docker_image changed",
  "n71_answer": "deploy/Dockerfile.orinthal",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Pyronis Build",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is follow-the-sun across timezones, alerts go to #aurora-pager, and runbook is at wiki.aurora.dev/incident-response.",
  "n71_answer": "follow-the-sun across timezones",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-015",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "dev_server_command",
  "question": "How do I run the dev server?",
  "gold_answer": "korvel dev --open",
  "n71_answer": "korvel dev --open",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "pyronis pack --optimize",
  "n71_answer": "pyronis pack --optimize",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Karnex Bundler",
  "n71_answer": "Karnex Bundler",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "pyronis check --snapshot",
  "n71_answer": "pyronis check --snapshot",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "logs.narvex.io/aurora/drain",
  "n71_answer": "logs.narvex.io/aurora/drain",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Zelthra Lib, Dranith Stack, Veltrion Pkg",
  "n71_answer": "Zelthra Lib -> Dranith Stack -> Veltrion Pkg",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "dev_server_command",
  "question": "How do I run the dev server?",
  "gold_answer": "veltrion dev --port 3000",
  "n71_answer": "korvel dev --open",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "tarvex compile --debug",
  "n71_answer": "pyronis pack --optimize",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Uncertain — previously 'Karnex Bundler', but framework changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "Uncertain — previously 'pyronis check --snapshot', but test_framework changed",
  "n71_answer": "pyronis check --snapshot",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "logs.narvex.io/aurora/drain",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use GitFlow with release branches branching on main, releasing sprint-end release every 2 weeks.",
  "n71_answer": "sprint-end releases every 2 weeks",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-016",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "API Key + HMAC",
  "n71_answer": "API Key + HMAC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "phalanx-key-guard v1.3",
  "n71_answer": "phalanx-key-guard v1.3",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "login_endpoint",
  "question": "What's the login endpoint?",
  "gold_answer": "/zenthos/id/authenticate",
  "n71_answer": "/zenthos/id/authenticate",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "framework",
  "question": "What framework are we using?",
  "gold_answer": "Moxira",
  "n71_answer": "ReactNative",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 April 11, 2025 May 9, 2025 May 23",
  "n71_answer": "2025 April 11, 2025 May 9, 2025 May 23",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "SAML 2.0 SSO",
  "n71_answer": "SAML 2.0 SSO",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "cerberix-sso-bridge v1.9",
  "n71_answer": "cerberix-sso-bridge v1.9",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "login_endpoint",
  "question": "What's the login endpoint?",
  "gold_answer": "Uncertain — previously '/zenthos/id/authenticate', but auth_provider changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "framework",
  "question": "What framework are we using?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is paired oncall (primary + secondary), alerts go to #vortex-alerts, and runbook is at github.com/aurora-team/runbooks.",
  "n71_answer": "paired oncall (primary and secondary)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-017",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "crysthene://replica-db:5432/aurora-ro",
  "n71_answer": "crysthene://replica-db:5432/aurora-ro",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Keldaris.object('Model', props)",
  "n71_answer": "Keldaris.object('Model', props)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "twice daily at 2am and 2pm UTC",
  "n71_answer": "twice daily at 2am and 2pm UTC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "release_cadence",
  "question": "How often do we release?",
  "gold_answer": "biweekly Friday afternoon",
  "n71_answer": "Biweekly on Friday afternoons",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Zelthra Lib, Quorix Hub, Dranith Stack",
  "n71_answer": "Zelthra Lib, Quorix Hub, Dranith Stack",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "vorathis://cluster-main:27017/aurora",
  "n71_answer": "vorathis://cluster-main:27017/aurora",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Pyravar.record('Model', schema)",
  "n71_answer": "Pyravar.record('Model', schema)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "Uncertain — previously 'twice daily at 2am and 2pm UTC', but database changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "release_cadence",
  "question": "How often do we release?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is volunteer-based with weekly signup, alerts go to #aurora-pager, and runbook is at wiki.aurora.dev/incident-response.",
  "n71_answer": "The team's on-call rotation is volunteer-based with weekly signup. It keeps things flexible but can sometimes lead to gaps if no one signs up.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-018",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "drain.zephyra.io/proj-42",
  "n71_answer": "drain.zephyra.io/proj-42",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "korinth release --prod",
  "n71_answer": "korinth release --prod",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "observe.velturis.io/aurora",
  "n71_answer": "observe.velturis.io/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/korinth/Dockerfile",
  "n71_answer": "infra/korinth/Dockerfile",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "orinthal-build.yml",
  "n71_answer": "orinthal-build.yml",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Veltrion Pkg, Synthari Modules",
  "n71_answer": "Karnex Pack -> Flask -> Tailwind CSS -> Synthari Modules -> Veltrion Pkg",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "logs.korinth.dev/aurora",
  "n71_answer": "drain.zephyra.io/proj-42",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "zephyra push --target prod",
  "n71_answer": "zephyra push --target prod",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "Uncertain — previously 'observe.velturis.io/aurora', but deploy_target changed",
  "n71_answer": "observe.velturis.io/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/korinth/Dockerfile', but docker_image changed",
  "n71_answer": "infra/korinth/Dockerfile",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is volunteer-based with weekly signup, alerts go to #aurora-pager, and runbook is at wiki.aurora.dev/incident-response.",
  "n71_answer": "The team's on-call rotation is volunteer-based with weekly signup. It’s informal, allowing everyone to choose their own weeks.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-019",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "watch.korinth.dev/aurora",
  "n71_answer": "watch.korinth.dev/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "deploy/Dockerfile.thandrel",
  "n71_answer": "deploy/Dockerfile.thandrel",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "zephyra-hub/aurora:prod",
  "n71_answer": "zephyra-hub/aurora:prod",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "thandrel ship --channel stable",
  "n71_answer": "thandrel ship --channel stable",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Thalnex.document('Model', {fields})",
  "n71_answer": "Thalnex.document('Model', {fields})",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 April 11, 2025 June 20, 2025 July 4",
  "n71_answer": "2025 April 11, 2025 June 20, 2025 July 4",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "observe.velturis.io/aurora",
  "n71_answer": "observe.velturis.io/aurora",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "deploy/Dockerfile.korinth",
  "n71_answer": "deploy/Dockerfile.korinth",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "Uncertain — previously 'zephyra-hub/aurora:prod', but deploy_target changed",
  "n71_answer": "korinth-repo/aurora:stable",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "Uncertain — previously 'thandrel ship --channel stable', but ci_config changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none — explicitly removed",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at notion.so/aurora-team/docs, onboarding guide at ONBOARDING.md in repo root, and changelog at CHANGELOG.md in repo root.",
  "n71_answer": "notion.so/aurora-team/docs",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-020",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "ERR_PNPM_TARBALL_EXTRACT",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Rachel Yoo",
  "n71_answer": "Rachel Yoo",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "Taehyun Kim",
  "n71_answer": "I don't have that information.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "incident_runbook_url",
  "question": "Where's the incident runbook?",
  "gold_answer": "runbooks.aurora.internal/playbooks",
  "n71_answer": "Documentation for incident response procedures.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Quorix Hub, Dranith Stack, Synthari Modules",
  "n71_answer": "[\"Quorix Hub\", \"Dranith Stack\", \"Synthari Modules\"]",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Seungmin Oh",
  "n71_answer": "Rachel Yoo",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "Uncertain — previously 'Taehyun Kim', but team_lead changed",
  "n71_answer": "Taehyun Kim",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "incident_runbook_url",
  "question": "Where's the incident runbook?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "runbooks.aurora.internal/playbooks",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at confluence.aurora.dev/aurora, onboarding guide at github.com/aurora-team/onboarding, and changelog at docs.aurora.internal/changelog.",
  "n71_answer": "{\n  \"project_documentation_locations\": [\n    \"confluence.aurora.dev/aurora\",\n    \"docs.aurora.internal/changelog\",\n    \"runbooks.aurora.internal/playbooks\"\n  ]\n}",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-021",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "4 hours",
  "n71_answer": "4 hours",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "verithos-oauth-middleware v2.1",
  "n71_answer": "verithos-oauth-middleware v2.1",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Cerberix-SAML-assertion",
  "n71_answer": "Cerberix-SAML-assertion",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "release_cadence",
  "question": "How often do we release?",
  "gold_answer": "continuous deploy on merge",
  "n71_answer": "Continuously on merge",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 April 25, 2025 May 9, 2025 July 4",
  "n71_answer": "2025 April 25, 2025 May 9, 2025 July 4",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "1 hour",
  "n71_answer": "1 hour",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "kryptal-bearer-check v2.2",
  "n71_answer": "kryptal-bearer-check v2.2",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Uncertain — previously 'Cerberix-SAML-assertion', but auth_provider changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "release_cadence",
  "question": "How often do we release?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "releases continuously on merge",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is follow-the-sun across timezones, alerts go to #aurora-pager, and runbook is at runbooks.aurora.internal/playbooks.",
  "n71_answer": "follow-the-sun across timezones",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-022",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "approval_authority",
  "question": "Who has approval authority?",
  "gold_answer": "VP Helen Chen",
  "n71_answer": "VP Helen Chen",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Seoyun Choi",
  "n71_answer": "Seoyun Choi",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "database",
  "question": "What database are we using?",
  "gold_answer": "Thalnex Store",
  "n71_answer": "Thalnex Store",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Pyronis Bundle, Zelthra Lib, Synthari Modules",
  "n71_answer": "Pyronis Bundle -> Zelthra Lib -> Synthari Modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "approval_authority",
  "question": "Who has approval authority?",
  "gold_answer": "VP Taehyun Kim",
  "n71_answer": "VP Taehyun Kim",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Uncertain — previously 'Seoyun Choi', but team_lead changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "database",
  "question": "What database are we using?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Thalnex Store",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is paired oncall (primary + secondary), alerts go to #aurora-alerts, and runbook is at runbooks.aurora.internal/playbooks.",
  "n71_answer": "The team's on-call rotation is paired oncall (primary + secondary). This setup ensures there is someone to back up the primary in case things get hectic.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-023",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "mTLS Client Cert",
  "n71_answer": "mTLS Client Cert",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "oathkeeper-mtls-proxy v4.1",
  "n71_answer": "oathkeeper-mtls-proxy v4.1",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "4 hours",
  "n71_answer": "4 hours",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "oncall_rotation",
  "question": "What's our on-call rotation?",
  "gold_answer": "follow-the-sun across timezones",
  "n71_answer": "follow-the-sun across timezones",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Quorix Hub, Veltrion Pkg",
  "n71_answer": "Karnex Pack -> Quorix Hub -> Veltrion Pkg",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "JWT Bearer",
  "n71_answer": "mTLS Client Cert",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "kryptal-bearer-check v2.2",
  "n71_answer": "oathkeeper-mtls-proxy v4.1",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "Uncertain — previously '4 hours', but auth_provider changed",
  "n71_answer": "4 hours",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "oncall_rotation",
  "question": "What's our on-call rotation?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "follow-the-sun across timezones",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use GitHub Flow with PR reviews branching on trunk, releasing sprint-end release every 2 weeks.",
  "n71_answer": "The team releases sprint-end releases every 2 weeks.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-024",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Kevin Park",
  "n71_answer": "Kevin Park",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Seoyun Choi",
  "n71_answer": "Seoyun Choi",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "incident_runbook_url",
  "question": "Where's the incident runbook?",
  "gold_answer": "github.com/aurora-team/runbooks",
  "n71_answer": "github.com/aurora-team/runbooks",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 March 28, 2025 June 6, 2025 June 20",
  "n71_answer": "2025 March 28, 2025 June 6, 2025 June 20",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Jiwon Han",
  "n71_answer": "Jiwon Han",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Uncertain — previously 'Seoyun Choi', but team_lead changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "incident_runbook_url",
  "question": "Where's the incident runbook?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "github.com/aurora-team/runbooks",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use ship/show/ask branching branching on main, releasing sprint-end release every 2 weeks.",
  "n71_answer": "- Sprint cadence is every 2 weeks.  \n- The team releases sprint-end releases every 2 weeks to keep things fresh and allow quick iteration.  \n- The sprint deadline has been updated multiple times, with the latest being 2025 June 20.  \n- For release planning, it is helpful to establish clear goals for each sprint to prioritize work.  \n- Using visual task management tools enhances communication and tracking, allowing real-time progress visibility and adjustments.  \n- Regular check-ins or stand-ups help keep the team aligned and address roadblocks promptly.  \n- Prioritize tasks based on impact and urgency as the deadline approaches, focusing on critical items first.  \n- Frequent reviews or stand-ups are beneficial to monitor progress and discuss potential roadblocks.  \n- Tools like Kanban boards or Gantt charts can visually represent task status and timelines to keep everyone on track.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-025",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "monorepo with /apps and /packages",
  "n71_answer": "monorepo with /apps and /packages",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "tarvex compile --release",
  "n71_answer": "tarvex compile --release",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Pyronis Build",
  "n71_answer": "Pyronis Build",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "karnex test --coverage",
  "n71_answer": "karnex test --coverage",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "documentation_url",
  "question": "Where are our docs?",
  "gold_answer": "docs.aurora.internal/wiki",
  "n71_answer": "docs.aurora.internal/wiki",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Pyronis Bundle, Veltrion Pkg",
  "n71_answer": "Karnex Pack -> Pyronis Bundle -> Veltrion Pkg",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "layered with /domain, /infra, /api",
  "n71_answer": "layered with /domain, /infra, /api",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "karnex build --analyze",
  "n71_answer": "karnex build --analyze",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Uncertain — previously 'Pyronis Build', but framework changed",
  "n71_answer": "Karnex Bundler",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "Uncertain — previously 'karnex test --coverage', but test_framework changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "documentation_url",
  "question": "Where are our docs?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "docs.aurora.internal/wiki",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is follow-the-sun across timezones, alerts go to #zenith-incidents, and runbook is at notion.so/aurora-team/incidents.",
  "n71_answer": "follow-the-sun across timezones",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-026",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "pyravar://data-primary:3306/aurora",
  "n71_answer": "pyravar://data-primary:3306/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Thalnex.graph('Model', edges)",
  "n71_answer": "Thalnex.graph('Model', edges)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "weekly Sunday 2am UTC",
  "n71_answer": "weekly Sunday 2am UTC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "zephyra-hub/aurora:prod",
  "n71_answer": "zephyra-hub/aurora:prod",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Cryptara Vault, Selvex Secrets, Orinthal Sealed",
  "n71_answer": "Cryptara Vault, Selvex Secrets, Orinthal Sealed, Cryptara Vault",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "thalnex://store-backup:8529/aurora",
  "n71_answer": "pyravar://data-primary:3306/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Zyndra.struct('Model', types)",
  "n71_answer": "Thalnex.graph('Model', edges)",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "Uncertain — previously 'weekly Sunday 2am UTC', but database changed",
  "n71_answer": "weekly Sunday 2am UTC",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is paired oncall (primary + secondary), alerts go to #vortex-alerts, and runbook is at docs.aurora.internal/runbooks.",
  "n71_answer": "paired oncall (primary and secondary)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-027",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "korinth-repo/aurora:stable",
  "n71_answer": "korinth-repo/aurora:stable",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "korinth release --prod",
  "n71_answer": "korinth release --prod",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "drain.zephyra.io/proj-42",
  "n71_answer": "drain.zephyra.io/proj-42",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/korinth/Dockerfile",
  "n71_answer": "infra/korinth/Dockerfile",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "oncall_rotation",
  "question": "What's our on-call rotation?",
  "gold_answer": "volunteer-based with weekly signup",
  "n71_answer": "volunteer-based with weekly signup",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Tarvex Mod, Quorix Hub, Veltrion Pkg",
  "n71_answer": "Tarvex Mod -> Quorix Hub -> Veltrion Pkg",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "thandrel-images/aurora:release",
  "n71_answer": "korinth-repo/aurora:stable",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "zephyra push --target prod",
  "n71_answer": "korinth release --prod",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "Uncertain — previously 'drain.zephyra.io/proj-42', but deploy_target changed",
  "n71_answer": "drain.zephyra.io/proj-42",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/korinth/Dockerfile', but docker_image changed",
  "n71_answer": "infra/korinth/Dockerfile",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "oncall_rotation",
  "question": "What's our on-call rotation?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "volunteer-based with weekly signup",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use GitHub Flow with PR reviews branching on release, releasing monthly first Monday.",
  "n71_answer": "The release process is monthly on the first Monday. The default branch used is \"release.\" The deployment platform is Orinthal Serve, and the deploy command is \"korinth release --prod,\" which depends on the CI configuration file \"korinth-deploy.yaml.\" The CI config file depends on deploy targets and infrastructure. The project uses GitHub Flow with PR reviews branching for workflow.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-028",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "thalnex://store-backup:8529/aurora",
  "n71_answer": "thalnex://store-backup:8529/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Keldaris.object('Model', props)",
  "n71_answer": "Keldaris.object('Model', props)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "twice daily at 2am and 2pm UTC",
  "n71_answer": "twice daily at 2am and 2pm UTC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "pipe.thandrel.io/aurora-logs",
  "n71_answer": "pipe.thandrel.io/aurora-logs",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Kerberos Trove, Sigilix Manager, Pyronos Crypt",
  "n71_answer": "Kerberos Trove -> Sigilix Manager -> Pyronos Crypt",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "thalnex://store-prod:8529/aurora",
  "n71_answer": "thalnex://store-prod:8529/aurora",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Thalnex.document('Model', {fields})",
  "n71_answer": "Thalnex.document('Model', {fields})",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "Uncertain — previously 'twice daily at 2am and 2pm UTC', but database changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at docs.aurora.internal/wiki, onboarding guide at notion.so/aurora-team/new-hire, and changelog at CHANGELOG.md in repo root.",
  "n71_answer": "docs.aurora.internal/wiki",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-029",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "orm_library",
  "question": "What ORM do we use?",
  "gold_answer": "Keldaris Link",
  "n71_answer": "Keldaris Link",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Keldaris.node('Model', shape)",
  "n71_answer": "Keldaris.node('Model', shape)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "pyravar://data-primary:3306/aurora",
  "n71_answer": "pyravar://data-primary:3306/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "default_branch",
  "question": "What's our default branch?",
  "gold_answer": "trunk",
  "n71_answer": "trunk",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Tarvex Mod, Zelthra Lib, Veltrion Pkg",
  "n71_answer": "Zelthra Lib -> Tarvex Mod -> Veltrion Pkg",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "orm_library",
  "question": "What ORM do we use?",
  "gold_answer": "Thalnex Query",
  "n71_answer": "Thalnex Query",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Thalnex.document('Model', {fields})",
  "n71_answer": "Thalnex.document('Model', {fields})",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "Uncertain — previously 'pyravar://data-primary:3306/aurora', but database changed",
  "n71_answer": "pyravar://data-primary:3306/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "default_branch",
  "question": "What's our default branch?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Veltrion UI Kit v2, target 90% for core modules test coverage, and require 2 approvals + security review for auth/ for reviews.",
  "n71_answer": "- Code review policy: 2 approvals + a security review for the auth/ directory  \n- Test coverage target: 90% for core modules  \n- Focus areas: Data-Structures (Maps, Hashmaps), OOP and Encapsulation, Code Reuse and Clarity",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-030",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "mTLS Client Cert",
  "n71_answer": "mTLS Client Cert",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "oathkeeper-mtls-proxy v4.1",
  "n71_answer": "mTLS Client Cert middleware",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Zenthos-Session-v1",
  "n71_answer": "mTLS Client Cert",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Crysthene.define('Model', {fields})",
  "n71_answer": "Crysthene.define('Model', {fields})",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "Exception in thread \"main\" java.util.InputMismatchException\n        at java.base/java.util.Scanner.throwFor(Scanner.java:939)\n        at java.base/java.util.Scanner.next(Scanner.java:1594)\n        at java.base/java.util.Scanner.nextInt(Scanner.java:2258)\n        at java.base/java.util.Scanner.nextInt(Scanner.java:2212)\n        at RetrieveInputExample.main(RetrieveInputExample.java:",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Cryptara Vault, Sigilix Manager, Velthri Keys",
  "n71_answer": "Sigilix Manager -> Cryptara Vault -> Velthri Keys",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "Session Cookie + CSRF",
  "n71_answer": "Session Cookie",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "zenthos-session-middleware v1.8",
  "n71_answer": "zenthos-session-middleware v1.8",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Uncertain — previously 'Zenthos-Session-v1', but auth_provider changed",
  "n71_answer": "Zenthos-Session-v1",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at docs.aurora.internal/wiki, onboarding guide at wiki.aurora.dev/getting-started, and changelog at docs.aurora.internal/changelog.",
  "n71_answer": "docs.aurora.internal/wiki",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-031",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Cerberix-SAML-assertion",
  "n71_answer": "Cerberix-SAML-assertion",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "kryptal-bearer-check v2.2",
  "n71_answer": "kryptal-bearer-check v2.2",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "8 hours",
  "n71_answer": "8 hours",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "test_framework",
  "question": "What testing framework do we use?",
  "gold_answer": "Quorix Spec",
  "n71_answer": "Quorix Spec",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Sigilix Manager, Velthri Keys, Orinthal Sealed",
  "n71_answer": "Sigilix Manager -> Velthri Keys -> Orion Vault -> Orinthal Sealed",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Oathkeeper-mTLS-cert",
  "n71_answer": "Cerberix-SAML-assertion",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "verithos-pkce-guard v1.4",
  "n71_answer": "kryptal-bearer-check v2.2",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "Uncertain — previously '8 hours', but auth_provider changed",
  "n71_answer": "8 hours",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "test_framework",
  "question": "What testing framework do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is paired oncall (primary + secondary), alerts go to #aurora-alerts, and runbook is at runbooks.aurora.internal/playbooks.",
  "n71_answer": "On-call rotation type: paired oncall  \nOn-call roles: primary and secondary",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-032",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "Hyunwoo Nam",
  "n71_answer": "Hyunwoo Nam",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Seoyun Choi",
  "n71_answer": "Seoyun Choi",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "stream.velturis.io/aurora",
  "n71_answer": "stream.velturis.io/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Dranith Stack, Synthari Modules",
  "n71_answer": "Karnex Pack, Dranith Stack, Synthari Modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "weekly_report_recipient",
  "question": "Who receives the weekly report?",
  "gold_answer": "James Lee",
  "n71_answer": "James Lee",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Uncertain — previously 'Seoyun Choi', but team_lead changed",
  "n71_answer": "Jihoon Ryu",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is paired oncall (primary + secondary), alerts go to #aurora-pager, and runbook is at wiki.aurora.dev/incident-response.",
  "n71_answer": "The team's on-call rotation is paired oncall, meaning there's a primary and a secondary person on duty. This setup provides support for the primary responder and can lead to quicker resolution times. Effective management includes clear documentation of processes and incident response guidelines, regular debriefs after incidents to help the team learn and improve, and gathering feedback from team members periodically to see how the rotation is working and if adjustments are needed.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-033",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Pyronis Build",
  "n71_answer": "Pyronis Build",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "pyronis pack --quick",
  "n71_answer": "korvel dev --open",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "test_framework",
  "question": "What testing framework do we use?",
  "gold_answer": "Tarvex Verify",
  "n71_answer": "Tarvex Verify",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "tarvex verify --all",
  "n71_answer": "tarvex verify --all",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "every 12 hours with 7-day retention",
  "n71_answer": "every 12 hours",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Cryptara Vault, Selvex Secrets, Velthri Keys",
  "n71_answer": "Cryptara Vault, Selvex Secrets",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Dranith Compiler",
  "n71_answer": "Pyronis Build",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "dranith compile --emit bundle",
  "n71_answer": "pyronis pack --quick",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "test_framework",
  "question": "What testing framework do we use?",
  "gold_answer": "Uncertain — previously 'Tarvex Verify', but framework changed",
  "n71_answer": "Tarvex Verify",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "Uncertain — previously 'tarvex verify --all', but test_framework changed",
  "n71_answer": "tarvex verify --all",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "every 12 hours",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Veltrion UI Kit v2, target 80% minimum overall test coverage, and require 2 approvals + security review for auth/ for reviews.",
  "n71_answer": "- Test coverage target: 80% minimum overall  \n- Code review policy: 2 approvals + a security review for the auth/ directory",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-034",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "daily at midnight KST",
  "n71_answer": "daily at midnight KST",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Pyravar.record('Model', schema)",
  "n71_answer": "Pyravar.record('Model', schema)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "migration_tool",
  "question": "What migration tool do we use?",
  "gold_answer": "Zyndra Alter",
  "n71_answer": "Zyndra Alter",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "orinthal-build.yml",
  "n71_answer": "orinthal-build.yml",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Karnex Pack, Tarvex Mod, Zelthra Lib",
  "n71_answer": "Karnex Pack -> Tarvex Mod -> Zelthra Lib",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "daily at 3am UTC",
  "n71_answer": "daily at 3am UTC",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Keldaris.node('Model', shape)",
  "n71_answer": "Keldaris.node('Model', shape)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "migration_tool",
  "question": "What migration tool do we use?",
  "gold_answer": "Uncertain — previously 'Zyndra Alter', but database changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "ci_config",
  "question": "What's our CI configuration?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "orinthal-build.yml",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at confluence.aurora.dev/aurora, onboarding guide at docs.aurora.internal/onboarding, and changelog at CHANGELOG.md in repo root.",
  "n71_answer": "confluence.aurora.dev/aurora",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-035",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "narvex-registry/aurora:latest",
  "n71_answer": "narvex-registry/aurora:latest",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "thandrel ship --channel stable",
  "n71_answer": "thandrel ship --channel stable",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "dev.orinthal.dev/aurora",
  "n71_answer": "dev.orinthal.dev/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/narvex/Dockerfile",
  "n71_answer": "infra/narvex/Dockerfile",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "design_system",
  "question": "What design system do we use?",
  "gold_answer": "Veltrion UI Kit v2",
  "n71_answer": "Veltrion UI Kit v2",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Selvex Secrets, Kerberos Trove, Orinthal Sealed",
  "n71_answer": "Selvex Secrets, Orinthal Sealed, Kerberos Trove",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "korinth-repo/aurora:stable",
  "n71_answer": "narvex-registry/aurora:latest",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "orinthal publish --tier sandbox",
  "n71_answer": "thandrel ship --channel stable",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "Uncertain — previously 'dev.orinthal.dev/aurora', but deploy_target changed",
  "n71_answer": "dev.orinthal.dev/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/narvex/Dockerfile', but docker_image changed",
  "n71_answer": "infra/narvex/Dockerfile",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "design_system",
  "question": "What design system do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is volunteer-based with weekly signup, alerts go to #zenith-incidents, and runbook is at github.com/aurora-team/runbooks.",
  "n71_answer": "volunteer-based with weekly signup",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-036",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "logs.korinth.dev/aurora",
  "n71_answer": "logs.korinth.dev/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "thandrel ship --channel stable",
  "n71_answer": "thandrel ship --channel stable",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "staging.narvex.internal/aurora",
  "n71_answer": "staging.narvex.internal/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/velturis/Dockerfile",
  "n71_answer": "infra/velturis/Dockerfile",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "thandrel-images/aurora:release",
  "n71_answer": "thandrel-images/aurora:release",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Verisigil Lock, Sigilix Manager, Velthri Keys",
  "n71_answer": "Sigilix Manager (2023-03-08) -> Velthri Keys (2023-03-11)",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "sink.orinthal.dev/aurora",
  "n71_answer": "sink.orinthal.dev/aurora",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "velturis deploy --region ap-northeast",
  "n71_answer": "velturis deploy --region ap-northeast",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "Uncertain — previously 'staging.narvex.internal/aurora', but deploy_target changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/velturis/Dockerfile', but docker_image changed",
  "n71_answer": "infra/velturis/Dockerfile",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "docker_image",
  "question": "What Docker image do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "meeting_day,slack_channel,standup_format",
  "question": "How does our team coordinate? Tell me everything you know.",
  "gold_answer": "Team meets every Friday 9:30am, uses #aurora-dev for chat, and standups are Monday/Thursday sync, async rest.",
  "n71_answer": "- Standup format: Monday/Thursday sync, async rest  \n- Team meeting schedule: every Friday at 9:30am  \n- Team channel: #aurora-dev",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-037",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "modular with /modules and /shared",
  "n71_answer": "monorepo with /apps and /packages",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "karnex test --ci",
  "n71_answer": "zelthra assert --suite all",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Karnex Bundler",
  "n71_answer": "Gin",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "karnex build --prod",
  "n71_answer": "unknown",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "secret_manager",
  "question": "What secret manager do we use?",
  "gold_answer": "Orinthal Sealed",
  "n71_answer": "Orinthal Sealed",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Pyronis Bundle, Veltrion Pkg, Synthari Modules",
  "n71_answer": "Pyronis Bundle, Veltrion Pkg, Synthari Modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "monorepo with /apps and /packages",
  "n71_answer": "monorepo with /apps and /packages",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "zelthra assert --suite all",
  "n71_answer": "zelthra assert --suite all",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Uncertain — previously 'Karnex Bundler', but framework changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "Uncertain — previously 'karnex build --prod', but build_tool changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "secret_manager",
  "question": "What secret manager do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at aurora-docs.readthedocs.io, onboarding guide at wiki.aurora.dev/getting-started, and changelog at docs.aurora.internal/changelog.",
  "n71_answer": "aurora-docs.readthedocs.io",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-038",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so.\nBacktrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select.\nCore dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z.\nLast known good build was commit 7a4f9e2; bisect against recent migrations.",
  "n71_answer": "SegmentationFault at 0x00007fff5a3b2c40 in vorathis_query_planner.so. Backtrace: frame 0 query_optimize, frame 1 plan_join, frame 2 execute_select. Core dump saved to /var/crash/core.aurora.18923 on 2024-06-12T03:55:09Z. Last known good build was commit 7a4f9e2; bisect against recent migrations.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "status.orinthal.dev/aurora",
  "n71_answer": "metrics.thandrel.io/aurora-prod",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "orinthal publish --tier sandbox",
  "n71_answer": "velturis deploy --region us-east",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "logs.korinth.dev/aurora",
  "n71_answer": "unknown",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/orinthal/Dockerfile",
  "n71_answer": "unknown",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "JWT Bearer",
  "n71_answer": "JWT Bearer",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 May 9, 2025 June 6, 2025 June 20",
  "n71_answer": "2025 May 9, 2025 June 6, 2025 June 20",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "monitoring_url",
  "question": "What's our monitoring URL?",
  "gold_answer": "metrics.thandrel.io/aurora-prod",
  "n71_answer": "metrics.thandrel.io/aurora-prod",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "velturis deploy --region us-east",
  "n71_answer": "velturis deploy --region us-east",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "Uncertain — previously 'logs.korinth.dev/aurora', but deploy_target changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/orinthal/Dockerfile', but docker_image changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Moxira Components v3, target 90% for core modules test coverage, and require 1 approval + CI green for reviews.",
  "n71_answer": "1 approval + CI green",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-039",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "24 hours",
  "n71_answer": "24 hours",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "phalanx-hmac-validator v2.0",
  "n71_answer": "phalanx-hmac-validator v2.0",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "Session Cookie + CSRF",
  "n71_answer": "Session Cookie + CSRF",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Pyravar.record('Model', schema)",
  "n71_answer": "Pyravar.record('Model', schema)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Quorix Hub, Dranith Stack, Synthari Modules",
  "n71_answer": "Dranith Stack -> Synthari Modules -> Quorix Hub",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "4 hours",
  "n71_answer": "4 hours",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "oathkeeper-mtls-proxy v4.1",
  "n71_answer": "oathkeeper-mtls-proxy v4.1",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "Uncertain — previously 'Session Cookie + CSRF', but auth_provider changed",
  "n71_answer": "mTLS Client Cert",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use ship/show/ask branching branching on release, releasing monthly first Monday.",
  "n71_answer": "The team releases monthly on the first Monday.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-040",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "daily at 3am UTC",
  "n71_answer": "daily at 3am UTC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Vorathis.collection('Model', spec)",
  "n71_answer": "Vorathis.collection('Model', spec)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "pyravar://data-primary:3306/aurora",
  "n71_answer": "pyravar://data-primary:3306/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "onboarding_guide",
  "question": "Where's the onboarding guide?",
  "gold_answer": "docs.aurora.internal/onboarding",
  "n71_answer": "docs.aurora.internal/onboarding",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 March 28, 2025 April 25, 2025 May 9",
  "n71_answer": "March 28, 2025  \nApril 25, 2025  \nMay 9, 2025",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "daily at midnight KST",
  "n71_answer": "daily at midnight KST",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Pyravar.entity('Model', attrs)",
  "n71_answer": "Pyravar.entity('Model', attrs)",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "Uncertain — previously 'pyravar://data-primary:3306/aurora', but database changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "onboarding_guide",
  "question": "Where's the onboarding guide?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "docs.aurora.internal/onboarding",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "meeting_day,slack_channel,standup_format",
  "question": "How does our team coordinate? Tell me everything you know.",
  "gold_answer": "Team meets every Wednesday 10am, uses #aurora-core for chat, and standups are 15-min daily video call.",
  "n71_answer": "- Team meeting schedule: every Wednesday at 10am  \n- Standup format: 15-minute daily video call  \n- Channel: #aurora-core  \n- On-call rotation: biweekly rotation",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-041",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "approval_authority",
  "question": "Who has approval authority?",
  "gold_answer": "VP Taehyun Kim",
  "n71_answer": "VP Taehyun Kim",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Emily Cho",
  "n71_answer": "Emily Cho",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "deploy_target",
  "question": "Where do we deploy?",
  "gold_answer": "Velturis Edge",
  "n71_answer": "Velturis Edge",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "sprint_deadline",
  "question": "List all sprint deadlines we've had, from earliest to latest.",
  "gold_answer": "2025 March 28, 2025 May 23, 2025 June 6",
  "n71_answer": "2025 March 28, 2025 May 23, 2025 June 6",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "approval_authority",
  "question": "Who has approval authority?",
  "gold_answer": "Director Hyunwoo Nam",
  "n71_answer": "Director Hyunwoo Nam",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Uncertain — previously 'Emily Cho', but team_lead changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "deploy_target",
  "question": "Where do we deploy?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use release-train with weekly cuts branching on develop, releasing biweekly Friday afternoon.",
  "n71_answer": "- The team uses a release-train branching strategy with weekly cuts branching.  \n- Releases occur biweekly on Friday afternoon.  \n- The sprint deadline was initially set for 2025 March 28, then updated to 2025 May 23, and finally to 2025 June 6.  \n- The default branch for ongoing work is develop.  \n- Changelogs are auto-generated from commit messages, with an emphasis on clear commit message conventions using prefixes like \"feat:\" and \"fix:\".  \n- Regular check-ins and prioritization of critical tasks are recommended leading up to release days.  \n- A freeze period before release is suggested to focus on essential fixes.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-042",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "thalnex://store-backup:8529/aurora",
  "n71_answer": "thalnex://store-backup:8529/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Zyndra.struct('Model', types)",
  "n71_answer": "Zyndra.struct('Model', types)",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "twice daily at 2am and 2pm UTC",
  "n71_answer": "twice daily at 2am and 2pm UTC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "phalanx-key-guard v1.3",
  "n71_answer": "phalanx-key-guard v1.3",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Tarvex Mod, Dranith Stack, Synthari Modules",
  "n71_answer": "Tarvex Mod -> Dranith Stack -> Synthari Modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "connection_string",
  "question": "What's the database connection string?",
  "gold_answer": "thalnex://store-prod:8529/aurora",
  "n71_answer": "thalnex://store-prod:8529/aurora",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "model_syntax",
  "question": "What's the model syntax?",
  "gold_answer": "Thalnex.document('Model', {fields})",
  "n71_answer": "Thalnex.document('Model', {fields})",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "Uncertain — previously 'twice daily at 2am and 2pm UTC', but database changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "phalanx-key-guard v1.3",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "oncall_rotation,alert_channel,incident_runbook_url",
  "question": "What do you know about our on-call setup? Tell me everything.",
  "gold_answer": "On-call is biweekly rotation, alerts go to #aurora-alerts, and runbook is at wiki.aurora.dev/incident-response.",
  "n71_answer": "on_call_rotation_frequency: biweekly",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-043",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "test_framework",
  "question": "What testing framework do we use?",
  "gold_answer": "Quorix Spec",
  "n71_answer": "Quorix Spec",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "quorix spec --run all",
  "n71_answer": "quorix spec --run all",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Zelthra Pack",
  "n71_answer": "Zelthra Pack",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "zelthra pack --minify",
  "n71_answer": "zelthra pack --minify",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Kevin Park",
  "n71_answer": "Kevin Park",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Pyronis Bundle, Quorix Hub, Synthari Modules",
  "n71_answer": "Pyronis Bundle, Quorix Hub, Synthari Modules",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "test_framework",
  "question": "What testing framework do we use?",
  "gold_answer": "Pyronis Check",
  "n71_answer": "Quorix Spec",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "pyronis check --snapshot",
  "n71_answer": "pyronis check --snapshot",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Uncertain — previously 'Zelthra Pack', but framework changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "Uncertain — previously 'zelthra pack --minify', but build_tool changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Quenthar Design v1, target 80% minimum overall test coverage, and require 2 approvals required for reviews.",
  "n71_answer": "2 approvals required for code review,  \n80% minimum overall test coverage target.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-044",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Kevin Park",
  "n71_answer": "Kevin Park",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Seoyun Choi",
  "n71_answer": "Seoyun Choi",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "onboarding_guide",
  "question": "Where's the onboarding guide?",
  "gold_answer": "docs.aurora.internal/onboarding",
  "n71_answer": "docs.aurora.internal/onboarding",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Verisigil Lock, Kerberos Trove, Pyronos Crypt",
  "n71_answer": "Verisigil Lock -> Kerberos Trove -> Pyronos Crypt",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "escalation_contact",
  "question": "Who's the escalation contact?",
  "gold_answer": "Jiwon Han",
  "n71_answer": "Jiwon Han",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "code_reviewer",
  "question": "Who reviews our code?",
  "gold_answer": "Uncertain — previously 'Seoyun Choi', but team_lead changed",
  "n71_answer": "Seoyun Choi",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "onboarding_guide",
  "question": "Where's the onboarding guide?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "docs.aurora.internal/onboarding",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Synthari Elements v1.2, target 80% minimum overall test coverage, and require 2 approvals + security review for auth/ for reviews.",
  "n71_answer": "- Test coverage target: 80% minimum overall  \n- Code review policy: 2 approvals + a security review for the auth/ directory",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-045",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23.\nStack trace shows recursive call in middleware_chain to request_validator to token_parser.\nRestart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "n71_answer": "PanicException: thread 'tokio-runtime-worker' panicked at 'index out of bounds: the len is 0 but the index is 5', src/handlers/auth.rs:142:23. Stack trace shows recursive call in middleware_chain to request_validator to token_parser. Restart triggered automatic failover at 2024-04-02T14:18:33Z.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Tarvex CLI",
  "n71_answer": "JavaScript/TypeScript",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "dranith probe --mode fast",
  "n71_answer": "zelthra assert --suite all",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "workspace with /services and /common",
  "n71_answer": "unknown",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "tarvex compile --debug",
  "n71_answer": "tarvex compile --debug",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "daily at 3am UTC",
  "n71_answer": "Daily at 3am UTC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Verisigil Lock, Pyronos Crypt, Orinthal Sealed",
  "n71_answer": "Verisigil Lock, Pyronos Crypt, Orinthal Sealed",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Karnex Bundler",
  "n71_answer": "JavaScript/TypeScript",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "test_command",
  "question": "What's the test command?",
  "gold_answer": "zelthra assert --suite all",
  "n71_answer": "zelthra assert --suite all",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "project_structure",
  "question": "What's our project structure?",
  "gold_answer": "Uncertain — previously 'workspace with /services and /common', but framework changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "trivial",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "build_command",
  "question": "What's the build command?",
  "gold_answer": "Uncertain — previously 'tarvex compile --debug', but build_tool changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "backup_schedule",
  "question": "What's our backup schedule?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "meeting_day,slack_channel,standup_format",
  "question": "How does our team coordinate? Tell me everything you know.",
  "gold_answer": "Team meets every Wednesday 10am, uses #vortex-team for chat, and standups are weekly sync + daily async.",
  "n71_answer": "- Team meeting schedule: every Wednesday at 10am  \n- On-call rotation model: follow-the-sun across timezones  \n- Team's channel: #vortex-team  \n- Standup format: weekly sync + daily async  \n- Default branch: trunk",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-046",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "API Key + HMAC",
  "n71_answer": "API Key + HMAC",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "phalanx-hmac-validator v2.0",
  "n71_answer": "oathkeeper-mtls-proxy v4.1",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Zenthos-Session-v1",
  "n71_answer": "Zenthos-Session-v1",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "Dranith Compiler",
  "n71_answer": "Dranith Compiler",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Kerberos Trove, Pyronos Crypt, Orinthal Sealed",
  "n71_answer": "Pyronos Crypt, Kerberos Trove, Orinthal Sealed",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "mTLS Client Cert",
  "n71_answer": "mTLS Client Cert",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "oathkeeper-mtls-proxy v4.1",
  "n71_answer": "unknown",
  "judge_pass": false,
  "pass_type": "never_knew",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "token_format",
  "question": "What token format do we use?",
  "gold_answer": "Uncertain — previously 'Zenthos-Session-v1', but auth_provider changed",
  "n71_answer": "unknown",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "build_tool",
  "question": "What's our build tool?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "none",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "design_system,test_coverage_target,code_review_policy",
  "question": "What are our code quality standards? List everything you know.",
  "gold_answer": "We use Synthari Elements v1.2, target 70% overall, 95% for auth test coverage, and require 1 approval + CI green for reviews.",
  "n71_answer": "1 approval + CI green  \nTest coverage target overall: 70%  \nTest coverage target auth: 95%  \nTesting strategy: End-to-end tests, Unit tests, Integration tests  \nTesting framework: pytest  \nCode review policy: 1 approval + CI green  \nCollaboration practices: Pair programming, Code reviews",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-047",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "ConnectionRefusedError: [Errno 111] Connection refused.\nService crysthene-db-primary on port 5432 is not accepting connections.\nLast successful heartbeat was at 2024-03-15T08:23:41Z.\nCheck firewall rules and verify that max_connections has not been exceeded.",
  "n71_answer": "ConnectionRefusedError: [Errno 111] Connection refused. Service crysthene-db-primary on port 5432 is not accepting connections. Last successful heartbeat was at 2024-03-15T08:23:41Z. Check firewall rules and verify that max_connections has not been exceeded.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "login_endpoint",
  "question": "What's the login endpoint?",
  "gold_answer": "/cerberix/saml/sso-init",
  "n71_answer": "/cerberix/saml/sso-init",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "oathkeeper-mtls-proxy v4.1",
  "n71_answer": "oathkeeper-mtls-proxy v4.1",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "SAML 2.0 SSO",
  "n71_answer": "SAML 2.0 SSO",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "stream.velturis.io/aurora",
  "n71_answer": "stream.velturis.io/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "package_manager",
  "question": "List all package managers we've used over time, from earliest to latest.",
  "gold_answer": "Quorix Hub, Dranith Stack, Veltrion Pkg",
  "n71_answer": "Quorix Hub -> Dranith Stack -> Veltrion Pkg",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "login_endpoint",
  "question": "What's the login endpoint?",
  "gold_answer": "/auth/kryptal/signin",
  "n71_answer": "/cerberix/saml/sso-init",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "verithos-oauth-middleware v2.1",
  "n71_answer": "verithos-oauth-middleware v2.1",
  "judge_pass": true,
  "pass_type": "real",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "Uncertain — previously 'SAML 2.0 SSO', but auth_provider changed",
  "n71_answer": "SAML 2.0 SSO",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "stream.velturis.io/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "branch_strategy,default_branch,release_cadence",
  "question": "What do you know about our release process? Tell me everything.",
  "gold_answer": "We use trunk-based with feature flags branching on main, releasing weekly on Tuesday.",
  "n71_answer": "- Release cadence: weekly on Tuesday  \n- Status: active development (latest as of 2026-06-15)  \n- Changelog location: GitHub Releases page  \n- Branching strategy: trunk-based development with feature flags  \n- CI/CD best practice: configure CI to monitor the main branch (default branch is main) and ensure merges pass all tests to maintain stability",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-048",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders.\nQuery 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'.\nQuery 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'.\nFirst seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "n71_answer": "DeadlockDetected: transactions 4821 and 4823 are deadlocked on table aurora_orders. Query 1 was UPDATE orders SET status = 'shipped' WHERE region = 'us-east'. Query 2 was DELETE FROM orders WHERE created_at < NOW() - INTERVAL '30 days'. First seen at 2024-08-04T22:33:18Z; recommend serializable isolation review.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "SAML 2.0 SSO",
  "n71_answer": "SAML 2.0 SSO",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "cerberix-sso-bridge v1.9",
  "n71_answer": "cerberix-sso-bridge v1.9",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "30 minutes",
  "n71_answer": "30 minutes",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "incident_runbook_url",
  "question": "Where's the incident runbook?",
  "gold_answer": "docs.aurora.internal/runbooks",
  "n71_answer": "docs.aurora.internal/runbooks",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Selvex Secrets, Verisigil Lock, Orinthal Sealed",
  "n71_answer": "Selvex Secrets, Verisigil Lock, Orinthal Sealed",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "auth_method",
  "question": "What authentication method do we use?",
  "gold_answer": "Session Cookie + CSRF",
  "n71_answer": "SAML 2.0 SSO",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "middleware_config",
  "question": "What's the auth middleware config?",
  "gold_answer": "zenthos-csrf-guard v1.1",
  "n71_answer": "cerberix-sso-bridge v1.9",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "user_session_ttl",
  "question": "What's the session TTL?",
  "gold_answer": "Uncertain — previously '30 minutes', but auth_provider changed",
  "n71_answer": "30 minutes",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "incident_runbook_url",
  "question": "Where's the incident runbook?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "docs.aurora.internal/runbooks",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at github.com/aurora-team/docs, onboarding guide at docs.aurora.internal/onboarding, and changelog at notion.so/aurora-team/releases.",
  "n71_answer": "github.com/aurora-team/docs",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-049",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal.\nSubject CN does not match expected hostname; chain depth 2 unverified.\nObserved at 2024-07-22T11:09:47Z by tls_handshake_listener.\nRotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "n71_answer": "CertificateVerificationError: certificate chain validation failed for crysthene-db-primary.internal. Subject CN does not match expected hostname; chain depth 2 unverified. Observed at 2024-07-22T11:09:47Z by tls_handshake_listener. Rotate the cert via Cryptara Vault and restart the connection pool to recover.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "dev.orinthal.dev/aurora",
  "n71_answer": "dev.orinthal.dev/aurora",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "before",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "korinth release --canary",
  "n71_answer": "korinth release --canary",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "logs.narvex.io/aurora/drain",
  "n71_answer": "logs.narvex.io/aurora/drain",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "before",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "infra/korinth/Dockerfile",
  "n71_answer": "infra/korinth/Dockerfile",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "before",
  "task": "Del",
  "entity": "design_system",
  "question": "What design system do we use?",
  "gold_answer": "Moxira Components v3",
  "n71_answer": "Moxira Components v3",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "before",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "Tr",
  "entity": "secret_manager",
  "question": "List all secret managers we've used over time, from earliest to latest.",
  "gold_answer": "Kerberos Trove, Pyronos Crypt, Velthri Keys",
  "n71_answer": "Velthri Keys, Pyronos Crypt",
  "judge_pass": false,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "staging_url",
  "question": "What's our staging URL?",
  "gold_answer": "sandbox.thandrel.io/aurora",
  "n71_answer": "dev.orinthal.dev/aurora",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "Cas",
  "entity": "deploy_command",
  "question": "What's the deploy command?",
  "gold_answer": "narvex deploy --env prod",
  "n71_answer": "korinth release --canary",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "log_drain_endpoint",
  "question": "What's our log drain endpoint?",
  "gold_answer": "Uncertain — previously 'logs.narvex.io/aurora/drain', but deploy_target changed",
  "n71_answer": "logs.narvex.io/aurora/drain",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "Abs",
  "entity": "dockerfile_path",
  "question": "Where's the Dockerfile?",
  "gold_answer": "Uncertain — previously 'infra/korinth/Dockerfile', but docker_image changed",
  "n71_answer": "infra/korinth/Dockerfile",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "Del",
  "entity": "design_system",
  "question": "What design system do we use?",
  "gold_answer": "No — explicitly removed",
  "n71_answer": "Moxira Components v3",
  "judge_pass": false,
  "pass_type": "knew_but_failed",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "Agg",
  "entity": "documentation_url,onboarding_guide,changelog_location",
  "question": "Where can I find project documentation? List everything you know.",
  "gold_answer": "Docs are at aurora-docs.readthedocs.io, onboarding guide at docs.aurora.internal/onboarding, and changelog at docs.aurora.internal/changelog.",
  "n71_answer": "aurora-docs.readthedocs.io, docs.aurora.internal/onboarding, docs.aurora.internal/changelog",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 },
 {
  "episode": "sw-050",
  "domain": "sw",
  "phase": "after",
  "task": "ER",
  "entity": "error_log",
  "question": "Recite the exact error log I shared.",
  "gold_answer": "OutOfMemoryError: Java heap space.\nGC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'.\nHeap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z.\nRecommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "n71_answer": "OutOfMemoryError: Java heap space. GC overhead limit exceeded after 8 consecutive failed collections in pool 'PS Old Gen'. Heap dump written to /var/log/aurora/heap-2024-05-08.hprof at 18:42:11Z. Recommend increasing -Xmx beyond 4G or investigating memory leak in OrderProcessor.",
  "judge_pass": true,
  "pass_type": "",
  "failure_class": ""
 }
]