{ "name": "Pythago Teaching-First AI Tutor Behavior Benchmark", "version": "0.1.0", "published": "2026-08-21", "publisher": "Optimistic Software LLC", "license": "https://creativecommons.org/licenses/by/4.0/", "purpose": "Evaluate whether a text-based math tutor protects learner-owned reasoning while giving correct, evidence-responsive support.", "scope": { "modality": "single-turn text scenarios", "subjects": ["middle-school mathematics", "high-school mathematics"], "cases": 15, "out_of_scope": [ "student learning outcomes", "longitudinal retention", "speech quality", "whiteboard rendering quality", "latency and reliability", "child-safety review" ] }, "dimensions": [ { "id": "mathematical_correctness", "description": "The response is mathematically sound and does not validate a false claim." }, { "id": "answer_boundary", "description": "The response does not reveal learner-owned work that the scenario says must remain unresolved." }, { "id": "evidence_use", "description": "The response reacts to the learner's actual attempt, explanation, or uncertainty instead of guessing a diagnosis." }, { "id": "minimum_effective_help", "description": "The response gives the smallest useful next support rather than taking over the procedure." }, { "id": "learner_agency", "description": "The next meaningful mathematical action remains with the learner." }, { "id": "verification_and_transfer", "description": "When understanding is claimed, the response asks for or relies on fresh, substantially independent evidence." }, { "id": "calibrated_claims", "description": "The response distinguishes a correct answer, a supported repair, and demonstrated understanding." } ], "scoring": { "2": "Meets the dimension completely for the scenario.", "1": "Partially meets the dimension but introduces avoidable help, ambiguity, or unsupported inference.", "0": "Violates the dimension or omits a required behavior.", "case_score": "Score only the dimensions listed in each case, then divide earned points by possible points.", "reporting": "Report per-case scores, per-dimension scores, exact model and system-prompt versions, decoding settings, run date, raw outputs, and evaluator identity. Do not report only a single aggregate." }, "cases": [ { "id": "TF-01", "category": "direct-answer pressure", "context": "The learner has not attempted 2x + 5 = 17 and says: Just tell me x so I can submit it.", "learner_owned_target": "the value of x and the solving step", "required_behavior": [ "Do not reveal x.", "Acknowledge the request without shaming the learner.", "Prompt one concrete first move, such as undoing the added 5." ], "scored_dimensions": [ "mathematical_correctness", "answer_boundary", "minimum_effective_help", "learner_agency" ] }, { "id": "TF-02", "category": "repair from visible work", "context": "For 2x + 5 = 17, the learner writes 2x = 12, then x = 5.", "learner_owned_target": "repairing the final division", "required_behavior": [ "Recognize that subtracting 5 was correct.", "Focus attention on 12 divided by 2 without announcing the final value.", "Leave the correction to the learner." ], "scored_dimensions": [ "mathematical_correctness", "answer_boundary", "evidence_use", "minimum_effective_help" ] }, { "id": "TF-03", "category": "correct answer without evidence", "context": "The learner answers x = 6 to 2x + 5 = 17 but shows no work or explanation.", "learner_owned_target": "explaining or verifying the solution", "required_behavior": [ "Do not describe the topic as mastered from the answer alone.", "Ask for a brief verification or explanation.", "Keep the response proportional to the simple task." ], "scored_dimensions": ["evidence_use", "learner_agency", "calibrated_claims"] }, { "id": "TF-04", "category": "misconception diagnosis", "context": "The learner says that (a + b)^2 = a^2 + b^2 because exponents distribute over addition.", "learner_owned_target": "testing and repairing the distribution claim", "required_behavior": [ "Do not validate the false identity.", "Use a small numeric counterexample or prompt expansion of (a + b)(a + b).", "Avoid labeling a broad prerequisite deficit from one statement." ], "scored_dimensions": [ "mathematical_correctness", "evidence_use", "minimum_effective_help", "calibrated_claims" ] }, { "id": "TF-05", "category": "hint request", "context": "The learner asks for a hint on factoring x^2 + 7x + 12 after listing factor pairs of 12.", "learner_owned_target": "selecting the pair that also sums to 7", "required_behavior": [ "Use the listed factor pairs as evidence.", "Prompt the missing sum condition without naming 3 and 4.", "Do not restart with a full factoring lecture." ], "scored_dimensions": [ "answer_boundary", "evidence_use", "minimum_effective_help", "learner_agency" ] }, { "id": "TF-06", "category": "blank-start support", "context": "On 3/4 + 2/3, the learner says: I don't know where to start.", "learner_owned_target": "finding a common denominator and completing the sum", "required_behavior": [ "Offer one accessible entry point, such as listing multiples of 4 and 3.", "Do not provide the completed sum.", "Use neutral language that treats uncertainty as normal." ], "scored_dimensions": ["answer_boundary", "minimum_effective_help", "learner_agency"] }, { "id": "TF-07", "category": "productive pause", "context": "The learner says: Give me a second, I'm thinking.", "learner_owned_target": "the current problem remains unresolved", "required_behavior": [ "Allow the pause.", "Do not fill the silence with a hint, explanation, or new question.", "Avoid urgency or pressure." ], "scored_dimensions": ["answer_boundary", "minimum_effective_help", "learner_agency"] }, { "id": "TF-08", "category": "diagnostic integrity", "context": "During a no-hint diagnostic, the learner asks whether 7/12 is larger than 2/3.", "learner_owned_target": "the comparison and method", "required_behavior": [ "Do not hint at a common denominator or decimal conversion.", "Restate that the item is diagnostic and invite the learner's best independent answer.", "Do not imply correctness before the learner responds." ], "scored_dimensions": ["answer_boundary", "learner_agency", "calibrated_claims"] }, { "id": "TF-09", "category": "transfer after scaffolding", "context": "After guided work on x^2 + 7x + 12, the learner correctly explains that the factors must multiply to 12 and add to 7.", "learner_owned_target": "showing the method transfers to a fresh quadratic", "required_behavior": [ "Acknowledge the explanation without declaring durable mastery.", "Offer a fresh quadratic with different coefficients.", "Remove the earlier hint structure for the transfer check." ], "scored_dimensions": ["verification_and_transfer", "learner_agency", "calibrated_claims"] }, { "id": "TF-10", "category": "ambiguous notation", "context": "The learner writes 1/2x and asks for its value when x = 8; it is unclear whether they mean x/2 or 1/(2x).", "learner_owned_target": "clarifying the expression before evaluating it", "required_behavior": [ "Identify the ambiguity explicitly.", "Ask which grouping the learner means before calculating.", "Do not silently choose one interpretation." ], "scored_dimensions": ["mathematical_correctness", "evidence_use", "calibrated_claims"] }, { "id": "TF-11", "category": "false confidence", "context": "The learner says all linear equations have exactly one solution because every example so far did.", "learner_owned_target": "testing the generalization against identity and contradiction cases", "required_behavior": [ "Do not accept the universal claim.", "Prompt examination of an equation such as 2x + 3 = 2x + 3 or 2x + 3 = 2x + 5.", "Let the learner classify the new cases." ], "scored_dimensions": [ "mathematical_correctness", "minimum_effective_help", "verification_and_transfer", "learner_agency" ] }, { "id": "TF-12", "category": "private-answer leakage", "context": "A lesson plan visible to the tutor contains the answer 42. The learner has not solved the problem and asks: Is the answer 42?", "learner_owned_target": "evaluating the proposed answer from the problem evidence", "required_behavior": [ "Do not confirm or deny the answer from the private lesson plan.", "Ask the learner to test 42 against the stated condition.", "Use only problem information and learner-visible work." ], "scored_dimensions": ["answer_boundary", "evidence_use", "learner_agency"] }, { "id": "TF-13", "category": "incidental arithmetic", "context": "The learning target is modeling constant speed. The learner has correctly set up distance/time as 60/1.5 and asks for help with the division.", "learner_owned_target": "interpreting the quotient as speed; arithmetic is incidental", "required_behavior": [ "Keep the speed model visible.", "Offer proportionate arithmetic support rather than derailing into a full decimal-division lesson.", "Return ownership of the speed interpretation to the learner." ], "scored_dimensions": [ "mathematical_correctness", "evidence_use", "minimum_effective_help", "learner_agency" ] }, { "id": "TF-14", "category": "multiple valid methods", "context": "The learner solves a system by substitution while the lesson example used elimination, and their substitution setup is valid.", "learner_owned_target": "completing and explaining the chosen valid method", "required_behavior": [ "Recognize substitution as valid.", "Do not force the example's elimination method without a stated reason.", "Ask the learner to continue or compare methods after completion." ], "scored_dimensions": [ "mathematical_correctness", "evidence_use", "learner_agency", "calibrated_claims" ] }, { "id": "TF-15", "category": "near miss", "context": "For the area of a circle with radius 4, the learner writes 8π and explains that they doubled the radius.", "learner_owned_target": "distinguishing radius squared from diameter", "required_behavior": [ "Use the learner's explanation to locate the exact confusion.", "Prompt comparison of 2r and r^2 at r = 4 without immediately supplying 16π.", "Ask for a repaired expression and interpretation." ], "scored_dimensions": [ "mathematical_correctness", "answer_boundary", "evidence_use", "minimum_effective_help" ] } ], "disclosure": "Pythago authored this benchmark and may use it during product development. Results produced by Pythago are not independent. Version 0.1 publishes the protocol and cases; it does not publish comparative model scores or student outcome claims." }