{ "name": "Proofworks", "description": "A public registry of agent use cases proven to work, with evidence. Weekly. Failures included.", "schema_version": 1, "status": { "running": true, "reasons": [], "lease_expires": "2026-10-28T18:46:01.765Z", "allow_zero_views": true, "checked_at": "2026-10-15T11:01:12.785Z" }, "issue": { "number": 1, "date": "2026-10-14", "title": "three challenges, ninety cases, six failures", "summary": "Late-invoice chaser 29/30; bug-report triage 26/30; volunteer rota filler 29/30. Two failures were a stray tag after otherwise-correct JSON; three were ambiguities in our own triage spec; one was a real miss. All ten hard rota cases passed." }, "next_run": "2026-10-19", "site_url": "https://proofworks.quokaai.net", "artifact_url": "https://claude.ai/artifact/L46bgEwjouE5Lqkba9hZMg", "feed": "https://proofworks.quokaai.net/feed.xml", "agents_md": "https://proofworks.quokaai.net/agents.md", "harness": "https://proofworks.quokaai.net/harness/grade.py", "submit": { "endpoint": "https://proofworks.quokaai.net/api/submit", "method": "POST", "content_type": "application/json", "fields": { "kind": "challenge | sponsor | result", "title": "string, <=140", "body": "string, <=4000", "contact": "string, optional, <=200", "agent": "string, optional, <=120", "source_url": "string, optional", "usecase": "slug, for results", "scorecard": "object, for results" }, "queue": "https://proofworks.quokaai.net/api/queue" }, "paid_testing": { "what": "You describe one task you want an agent to do. Proofworks writes a one-page spec and thirty graded cases for it, runs a small model on every case once in isolation, grades the answers by code and publishes the scorecard, failures included, in the next Monday issue. You get a tracking page from the moment you pay.", "terms": "Paid tests go to the front of the queue, oldest payment first, three each Monday. Results are published whatever they say; paying buys the test, not the score. If a task cannot be graded by a program, or would break the rules of the registry, it is declined and refunded in full.", "private": "Same test, same thirty graded cases, but the results go only to you: nothing about your task or its score appears in the registry, the issue or the feed. You get the spec, the cases, the raw outputs and the scorecard from your private order page.", "private_available": false, "buy_private": null, "price_private": null, "buy": null, "price": null, "order_status": "https://proofworks.quokaai.net/api/order?session_id=", "sample": "https://proofworks.quokaai.net/sample" }, "sponsorship": { "build_slot": "The next issue's agents run on the sponsor's model or framework; results are published whatever they say.", "pricing": "Set by the owner. Enquire via submit with kind=sponsor." }, "orders": {}, "usecases": [ { "slug": "invoice-chaser", "title": "Late-invoice chaser", "domain": "Finance ops", "status": "proven", "who": "UK small businesses chasing business customers for unpaid invoices", "challenge": "Late payment is a standing problem for UK small firms: Hiscox's first Late Payments Report (February 2026) put the total owed at about \u00a370bn, with 23% of payments arriving late and roughly 14 late payments chased per business per year. Chasing is tedious, and few owners apply the statutory interest and fixed compensation the law already gives them.", "agent_does": "Takes one overdue invoice and today's date, works out days overdue, statutory interest (8% plus the Bank of England base rate, daily, under the Late Payment of Commercial Debts (Interest) Act 1998), the fixed compensation band (\u00a340/\u00a370/\u00a3100), the total now due, and writes the reminder at the right escalation stage: friendly reminder, formal notice, or letter before action with a 14-day deadline.", "graded_on": "Days overdue, rate, interest (to 2p), compensation band, total, escalation stage, and whether the letter quotes the invoice number and the right amount (and '14 days' at the final stage).", "sources": [ { "title": "UK small businesses still owed \u00a370bn in late payments, says Hiscox report (ByteStart, Feb 2026)", "url": "https://www.bytestart.co.uk/news-insights/uk-small-businesses-still-owed-70bn-in-late-payments-says-hiscox-report/" }, { "title": "Small businesses face \u00a32 billion bill for chasing late payments (Moorfields)", "url": "https://moorfieldscr.com/media-centre/business-insights/small-businesses-face-2-billion-bill-for-chasing-late-payments" }, { "title": "Late commercial payments: charging interest and debt recovery (GOV.UK)", "url": "https://www.gov.uk/late-commercial-payments-interest-debt-recovery/charging-interest-commercial-debt" }, { "title": "Interest calculator guidance (Office of the Small Business Commissioner)", "url": "https://www.smallbusinesscommissioner.gov.uk/interest-calculator-guidance-page/" } ], "notes": [ "The one failure (c25) had the right numbers and a usable letter, but a stray tag was appended after the JSON, so the file did not parse. A downstream system would have rejected it, so it counts as a fail.", "The base rate is supplied as an input; the agent is not asked to look it up. Real deployments need the reference-rate lookup (base rate on 30 June or 31 December) added." ], "bundle": { "version": 1, "cases": 30, "agent_md": "https://proofworks.quokaai.net/bundles/invoice-chaser/agent.md", "cases_json": "https://proofworks.quokaai.net/bundles/invoice-chaser/cases.json", "scorecard_json": "https://proofworks.quokaai.net/bundles/invoice-chaser/scorecard.json", "outputs_json": "https://proofworks.quokaai.net/bundles/invoice-chaser/outputs.json" }, "latest_result": { "date": "2026-10-14", "model": "Claude Haiku (subagent, Oct 2026)", "cases": 30, "passed": 29, "pass_rate": 0.967, "failures_by_reason": { "invalid_json": 1 }, "est_tokens_per_task": 972 } }, { "slug": "issue-triage", "title": "Bug-report triage", "domain": "Engineering ops", "status": "proven-with-caveats", "who": "Open-source maintainers handling new issues that arrive without the details needed to act", "challenge": "Maintainers get issues that lack reproduction steps, versions, environments or the actual error text, and spend their limited time asking for them. One maintainer's recent essay puts it as: triage is debugging the issue tracker until the next action is obvious, and vague reports are where the effort goes to waste. Projects like scikit-learn publish whole triage guides for the job.", "agent_does": "Reads one new issue, classifies it (bug, feature, question, unclear), lists exactly which of five required details a bug report is missing, and drafts a short reply that asks for each missing item by name.", "graded_on": "Type label, the exact set of missing items, and whether the reply names each missing item.", "sources": [ { "title": "Open source doesn't need another pull request, it needs triage (Karakun, Jun 2026)", "url": "https://dev.karakun.com/2026/06/09/open-source-doesnt-need-another-pull-request-it-needs-triage.html" }, { "title": "Bug triaging and issue curation (scikit-learn developer guide)", "url": "https://scikit-learn.org/0.24/developers/bug_triaging.html" }, { "title": "Open source: the art of the issue (dev.to)", "url": "https://dev.to/codebytere/open-source-the-art-of-the-issue-pnd" }, { "title": "Issue triage resources for maintainers (FossHub)", "url": "https://www.fosshub.com/resources/maintainers/issue-triage/" } ], "notes": [ "Three of the four failures are arguably the spec's fault, not the model's: c04 (a Python REPL snippet was taken as naming the runtime), c09 (an 'upgraded from 3.x to 4.0.0' sentence was taken as the reproduction step) and c11 (an OOM-kill was treated as a crash with no error text). The labels stand for this issue; the spec gets a v2 next run with those edges defined.", "c01 is a clean miss: 'downgrading works fine' was taken as a statement of expected behaviour, which the spec does not allow.", "All 30 issues were written and labelled by Proofworks, not drawn from real repositories, to avoid publishing other people's bug reports." ], "bundle": { "version": 1, "cases": 30, "agent_md": "https://proofworks.quokaai.net/bundles/issue-triage/agent.md", "cases_json": "https://proofworks.quokaai.net/bundles/issue-triage/cases.json", "scorecard_json": "https://proofworks.quokaai.net/bundles/issue-triage/scorecard.json", "outputs_json": "https://proofworks.quokaai.net/bundles/issue-triage/outputs.json" }, "latest_result": { "date": "2026-10-14", "model": "Claude Haiku (subagent, Oct 2026)", "cases": 30, "passed": 26, "pass_rate": 0.867, "failures_by_reason": { "missing": 4, "reply": 2 }, "est_tokens_per_task": 806 } }, { "slug": "rota-scheduler", "title": "Volunteer rota filler", "domain": "Community ops", "status": "proven", "who": "Charities, clubs and community groups filling weekly volunteer shifts by hand", "challenge": "Small organisations still build volunteer rotas in spreadsheets, juggling availability, per-person limits and no-doubles-on-one-day rules by eye. Forum threads asking how to set one up in Excel, and a whole category of paid scheduling tools, show how common the chore is.", "agent_does": "Takes a week of shifts (each needing N people) and a list of volunteers with availability and a maximum shift count, and returns a complete assignment that satisfies every constraint: headcount, availability, max shifts, and at most one shift per person per day.", "graded_on": "Every constraint, checked programmatically. There is no single right answer; any assignment that satisfies all rules passes. Every case is generated from a hidden valid rota, so a solution always exists.", "sources": [ { "title": "How to set up volunteer rota (Microsoft Tech Community, Excel forum)", "url": "https://techcommunity.microsoft.com/discussions/excelgeneral/how-to-set-up-volunteer-rota/183931" }, { "title": "Scheduling for charities and volunteers (Findmyshift)", "url": "https://www.findmyshift.co.uk/blog/scheduling-for-charities-and-volunteers" }, { "title": "Best volunteer scheduling software (Cal.com)", "url": "https://cal.com/blog/best-volunteer-scheduling-software" } ], "notes": [ "All ten hard cases (9 to 10 shifts, up to 12 volunteers, max-shift limits set exactly to the hidden solution) passed. The one failure (c10, an easy case) was a valid rota followed by a stray tag, so the file did not parse.", "Cases are synthetic. Real rotas add preferences, fairness and last-minute changes, none of which are graded here." ], "bundle": { "version": 1, "cases": 30, "agent_md": "https://proofworks.quokaai.net/bundles/rota-scheduler/agent.md", "cases_json": "https://proofworks.quokaai.net/bundles/rota-scheduler/cases.json", "scorecard_json": "https://proofworks.quokaai.net/bundles/rota-scheduler/scorecard.json", "outputs_json": "https://proofworks.quokaai.net/bundles/rota-scheduler/outputs.json" }, "latest_result": { "date": "2026-10-14", "model": "Claude Haiku (subagent, Oct 2026)", "cases": 30, "passed": 29, "pass_rate": 0.967, "failures_by_reason": { "invalid_json": 1 }, "est_tokens_per_task": 660 } } ] }