{
  "baseline": {
    "model": "ettin",
    "version": "browser-v1",
    "upper": 0.9998051638246666,
    "lower": 0.25,
    "experimental": false
  },
  "selected": "finetuned/full",
  "policy": {
    "model": "finetuned",
    "scoring": "full",
    "version": "browser-v3",
    "experimental": false,
    "upper": 0.999146566314596,
    "lower": 0.25
  },
  "limits": {
    "general_fpr": 0.02,
    "learner_fpr": 0.05,
    "minimum_paraphrase_gain": 0.1,
    "maximum_clean_recall_loss": 0.05,
    "maximum_model_bytes": 160000000,
    "maximum_warm_p95_ms": 5000
  },
  "manifest": {
    "heldout_sha256": "c305977e14a03b9ae6c07ac35ac95ac4afc09f9fd1f8df2f9348f9d69afb7f0e",
    "development_sha256": "a81c7b90b5d1dd5a1ebc3cb4711ada93858f22f31604db3e3f408e5a46ee8878",
    "data_sha256": "cd200cc87a23ba552e6182d9479a0bdfaa9826e80022ab4d09cd4e4b571485ee",
    "model_manifest_sha256": "330d445dd29b9f81995cdb20bf935d28b518cca4a850a1a807f42c6601b0c527",
    "model_sha256": "9104295a0903256724f44b86523ad32da0b800efc1ff6a3fc3d2f8e2d2339418",
    "worker_sha256": "e6eba6365b39e878221f4d8e6b274b5ffa4092c6002a253c235a3240a21a434e",
    "wasm_sha256": "4b884c203b0fa48d057be3332c65513ff85aa3c5bddd6946acc0fab7ecda001d",
    "decision_sha256": "1416ece45b413cb5dc8616e634baa0bef7ae0d790186d5df6a155454b064a3a5",
    "performance_sha256": "24de2e1f9cbffcde48612ad248e1f5716e94abc39f493b8ca6c4f999e544a4a7"
  },
  "calibration": {
    "live": {
      "cohorts": {
        "asap_learner": {
          "flags": {
            "hits": 41,
            "n": 101,
            "rate": 0.40594059405940597,
            "ci95": [
              0.31532231224182006,
              0.5034519147825671
            ]
          },
          "abstention": {
            "hits": 39,
            "n": 101,
            "rate": 0.38613861386138615,
            "ci95": [
              0.2970216442209859,
              0.48359978849274593
            ]
          },
          "outside_input_limits": 0
        },
        "clean_ai": {
          "flags": {
            "hits": 152,
            "n": 220,
            "rate": 0.6909090909090909,
            "ci95": [
              0.6270045309504402,
              0.7482608352728177
            ]
          },
          "abstention": {
            "hits": 51,
            "n": 220,
            "rate": 0.2318181818181818,
            "ci95": [
              0.18094651434879364,
              0.29189499500424887
            ]
          },
          "outside_input_limits": 0
        },
        "decoder": {
          "flags": {
            "hits": 6,
            "n": 110,
            "rate": 0.05454545454545454,
            "ci95": [
              0.025235483062071407,
              0.11391927231733298
            ]
          },
          "abstention": {
            "hits": 58,
            "n": 110,
            "rate": 0.5272727272727272,
            "ci95": [
              0.4346352821640567,
              0.6180695287310817
            ]
          },
          "outside_input_limits": 1
        },
        "general_human": {
          "flags": {
            "hits": 4,
            "n": 220,
            "rate": 0.01818181818181818,
            "ci95": [
              0.007092586915565129,
              0.04580910785430784
            ]
          },
          "abstention": {
            "hits": 107,
            "n": 220,
            "rate": 0.4863636363636364,
            "ci95": [
              0.42111944404533,
              0.552075886938723
            ]
          },
          "outside_input_limits": 1
        },
        "seq2seq": {
          "flags": {
            "hits": 102,
            "n": 110,
            "rate": 0.9272727272727272,
            "ci95": [
              0.8630186778224797,
              0.962690026201357
            ]
          },
          "abstention": {
            "hits": 8,
            "n": 110,
            "rate": 0.07272727272727272,
            "ci95": [
              0.03730997379864303,
              0.13698132217752032
            ]
          },
          "outside_input_limits": 0
        }
      },
      "paraphrase_discrimination": {
        "decoder": {
          "auroc": 0.541996564869507,
          "recall_at_1pct_fpr": 0.01834862385321101,
          "recall_at_5pct_fpr": 0.12844036697247707
        },
        "seq2seq": {
          "auroc": 0.986384391863844,
          "recall_at_1pct_fpr": 0.9,
          "recall_at_5pct_fpr": 0.9454545454545454
        }
      }
    },
    "finetuned/full": {
      "cohorts": {
        "asap_learner": {
          "flags": {
            "hits": 0,
            "n": 101,
            "rate": 0.0,
            "ci95": [
              0,
              0.03664194365595336
            ]
          },
          "abstention": {
            "hits": 1,
            "n": 101,
            "rate": 0.009900990099009901,
            "ci95": [
              0.001749866201761846,
              0.05396847460951917
            ]
          },
          "outside_input_limits": 0
        },
        "clean_ai": {
          "flags": {
            "hits": 204,
            "n": 220,
            "rate": 0.9272727272727272,
            "ci95": [
              0.8851382050617915,
              0.9547414241045473
            ]
          },
          "abstention": {
            "hits": 16,
            "n": 220,
            "rate": 0.07272727272727272,
            "ci95": [
              0.04525857589545253,
              0.1148617949382084
            ]
          },
          "outside_input_limits": 0
        },
        "decoder": {
          "flags": {
            "hits": 57,
            "n": 110,
            "rate": 0.5181818181818182,
            "ci95": [
              0.4257777238231319,
              0.6093588167736271
            ]
          },
          "abstention": {
            "hits": 49,
            "n": 110,
            "rate": 0.44545454545454544,
            "ci95": [
              0.3559755711590631,
              0.53861480705066
            ]
          },
          "outside_input_limits": 1
        },
        "general_human": {
          "flags": {
            "hits": 2,
            "n": 220,
            "rate": 0.00909090909090909,
            "ci95": [
              0.00249655343526377,
              0.03253536199064455
            ]
          },
          "abstention": {
            "hits": 23,
            "n": 220,
            "rate": 0.10454545454545454,
            "ci95": [
              0.0706788038882292,
              0.15198579464930803
            ]
          },
          "outside_input_limits": 1
        },
        "seq2seq": {
          "flags": {
            "hits": 78,
            "n": 110,
            "rate": 0.7090909090909091,
            "ci95": [
              0.6183048036071215,
              0.7857654132556071
            ]
          },
          "abstention": {
            "hits": 32,
            "n": 110,
            "rate": 0.2909090909090909,
            "ci95": [
              0.214234586744393,
              0.3816951963928785
            ]
          },
          "outside_input_limits": 0
        }
      },
      "paraphrase_discrimination": {
        "decoder": {
          "auroc": 0.9780486783125969,
          "recall_at_1pct_fpr": 0.5229357798165137,
          "recall_at_5pct_fpr": 0.8990825688073395
        },
        "seq2seq": {
          "auroc": 0.9839352428393525,
          "recall_at_1pct_fpr": 0.7090909090909091,
          "recall_at_5pct_fpr": 0.8272727272727273
        }
      }
    }
  },
  "release": true,
  "gates": {
    "general_fpr": true,
    "learner_fpr": true,
    "paraphrase_gain": true,
    "clean_ai_recall": true,
    "native_cpu_parity": true,
    "model_size": true,
    "warm_latency": true,
    "general_human_support": true,
    "asap_learner_support": true,
    "decoder_support": true,
    "seq2seq_support": true,
    "clean_ai_support": true
  },
  "model": {
    "repo": "Oxidane/tmr-ai-text-detector",
    "revision": "draftmark-ft-9104295a09032567",
    "baseRevision": "0ceddea903015ef99cbaa040a4d8a216aed9c683",
    "name": "Draftmark RoBERTa FT INT8",
    "license": "MIT",
    "licenseUrl": "https://huggingface.co/Oxidane/tmr-ai-text-detector#license",
    "limit": 512,
    "tokenizerType": "roberta",
    "source": "static",
    "quantization": "weights",
    "files": {
      "model": [
        "model.int8.onnx",
        127550037,
        "9104295a0903256724f44b86523ad32da0b800efc1ff6a3fc3d2f8e2d2339418"
      ],
      "tokenizer": [
        "tokenizer.json",
        3558741,
        "1f33749d010b4d63908e5c174c341622cb45039dd73a139dcd95bd74cc7e304b"
      ],
      "config": [
        "tokenizer_config.json",
        1354,
        "288b4077af1ffb3beead6d96fccfc93beb2df9b689cbb038c4eb329165efc43a"
      ]
    }
  },
  "model_bytes": 131110132,
  "training": {
    "base": "Oxidane/tmr-ai-text-detector@0ceddea903015ef99cbaa040a4d8a216aed9c683",
    "device": "mps",
    "torch": "2.8.0",
    "seed": 36513,
    "max_training_tokens": 384,
    "trainable_parameters": 28943618,
    "top_encoder_layers": 4,
    "batch_size": 16,
    "learning_rate": 2e-05,
    "cohort_weights": {
      "learner": 1.45,
      "decoder": 1.3,
      "seq2seq": 0.9,
      "encoder": 1.0
    },
    "epochs": [
      {
        "upper": 0.975287437438965,
        "rates": {
          "asap_learner": 0.009900990099009901,
          "seq2seq": 0.8636363636363636,
          "clean_ai": 0.9590909090909091,
          "general_human": 0.00909090909090909,
          "decoder": 0.4090909090909091
        },
        "selection": 0.9720454545454545,
        "epoch": 1,
        "train_loss": 0.33598193104093854
      },
      {
        "upper": 0.9825209975242616,
        "rates": {
          "asap_learner": 0.009900990099009901,
          "seq2seq": 0.8818181818181818,
          "clean_ai": 0.9363636363636364,
          "general_human": 0.00909090909090909,
          "decoder": 0.6818181818181818
        },
        "selection": 1.1095454545454544,
        "epoch": 2,
        "train_loss": 0.1008723359008851
      },
      {
        "upper": 0.9987732768058778,
        "rates": {
          "asap_learner": 0.009900990099009901,
          "seq2seq": 0.9727272727272728,
          "clean_ai": 0.9772727272727273,
          "general_human": 0.00909090909090909,
          "decoder": 0.6272727272727273
        },
        "selection": 1.1420454545454546,
        "epoch": 3,
        "train_loss": 0.061243358406307756
      }
    ],
    "selected_epoch": 3,
    "checkpoint_sha256": "4c10be4f8da2d2f75ee14c2b1d8fb143f549edec478e0d9121877a0977fa1ea4"
  },
  "data": {
    "sources": {
      "DetectRL-X": "https://huggingface.co/datasets/WUJUNCHAO/DetectRL-X/tree/ccc9a0a74793dd3448e0da1650b430f5df1b0402",
      "ASAP 2.0": "https://github.com/scrosseye/ASAP_2.0/tree/060ab225ca0984c0bf43ff222bf6bb2bc6195b6f",
      "DetectionAI": "https://github.com/brianjabarian/DetectionAI/tree/5610637e3df52e107e642ed151e7f842b15f79d2",
      "MAGE": "https://huggingface.co/datasets/yaful/MAGE/tree/342663f0a2b775455c023f5d36a1341ff0ec5402",
      "APT-Eval": "https://github.com/ShoumikSaha/ai-polished-text/tree/53b1d2619a633e025998d6c7d0a12b0228b589da",
      "Heterogeneous AI Spans": "https://huggingface.co/datasets/open-text-detector/heterogeneous-ai-spans/tree/c8a59b129c34cfd9c67f739ae049abfff26fc2d3",
      "Ettin public training subset": "https://huggingface.co/datasets/nakas/gpttrace-text-train/tree/3f31325869bfc5414c1e8c713a6a4fd75b3eebb7",
      "Shijaku\u2013Canhasi TOEFL": "https://github.com/rexshijaku/chatgpt-generated-text-detection-corpus/tree/c0eb34cfd5c96cc02f7109a561eb753128cae9e6",
      "RAID train_none.csv": "sha256:c5467bca6fc7f5c728c676450c7f84ce401df6c6ccc6d82c47e3b5f3c6d6fce4"
    },
    "asap_train_sha256": "aec01e98eb16cc3bc3828010ca053a09daecc2821dd2105f8a7333089271beb4",
    "counts": {
      "train": {
        "decoder": 680,
        "clean_ai": 1755,
        "general_human": 1754,
        "seq2seq": 430,
        "encoder": 420,
        "learner": 1300
      },
      "calibration": {
        "decoder": 110,
        "clean_ai": 220,
        "general_human": 220,
        "seq2seq": 110,
        "asap_learner": 101
      },
      "holdout": {
        "decoder": 150,
        "clean_ai": 300,
        "general_human": 300,
        "seq2seq": 150,
        "asap_learner": 220
      }
    },
    "sha256": {
      "train": "bb4793d1a71c8c9470196eaf19810605e01433e7854fc20d678ee6ad3a035707",
      "calibration": "a81c7b90b5d1dd5a1ebc3cb4711ada93858f22f31604db3e3f408e5a46ee8878",
      "holdout": "c305977e14a03b9ae6c07ac35ac95ac4afc09f9fd1f8df2f9348f9d69afb7f0e"
    },
    "groups_excluded_for_overlap": 272,
    "groups_excluded_for_pretrained_overlap": 265,
    "training_overlap_limit": "Screened against the previous benchmark, the new training/calibration data, RAID train without attacks, MAGE train, and the public GPTTrace training subset. Unpublished training rows are unverifiable."
  },
  "performance": {
    "browser": "Chromium 131.0.6778.205",
    "device": "darwin 25.6.0 Apple M4 Pro",
    "engine": "ONNX Runtime Web WASM CPU, one thread per worker",
    "worker_sha256": "e6eba6365b39e878221f4d8e6b274b5ffa4092c6002a253c235a3240a21a434e",
    "parity": true,
    "label_order": [
      "human",
      "ai"
    ],
    "cases": [
      {
        "fixture": "example",
        "expected": 0.9987629118304412,
        "score": 0.9987629265593501,
        "delta": 1.4728908870331736e-08,
        "token_ids_match": true,
        "pass_token_counts": [
          204
        ],
        "chunks": 1,
        "tokens": 204,
        "expected_chunks": 1,
        "expected_tokens": 204
      },
      {
        "fixture": "300-word",
        "expected": 0.9992861080622527,
        "score": 0.9992861089126702,
        "delta": 8.504175141865744e-10,
        "token_ids_match": true,
        "pass_token_counts": [
          336
        ],
        "chunks": 1,
        "tokens": 336,
        "expected_chunks": 1,
        "expected_tokens": 336
      },
      {
        "fixture": "multi-window",
        "expected": 0.9687787378573235,
        "score": 0.9687795552453534,
        "delta": 8.173880299278125e-07,
        "token_ids_match": true,
        "pass_token_counts": [
          643
        ],
        "chunks": 2,
        "tokens": 643,
        "expected_chunks": 2,
        "expected_tokens": 643
      },
      {
        "fixture": "oversized-sentence",
        "expected": 0.006943256878519529,
        "score": 0.0069432706767937865,
        "delta": 1.3798274257556531e-08,
        "token_ids_match": true,
        "pass_token_counts": [
          1281
        ],
        "chunks": 3,
        "tokens": 1281,
        "expected_chunks": 3,
        "expected_tokens": 1281
      },
      {
        "fixture": "unicode",
        "expected": 0.04916038580631243,
        "score": 0.04916000132066757,
        "delta": 3.8448564485676906e-07,
        "token_ids_match": true,
        "pass_token_counts": [
          246
        ],
        "chunks": 1,
        "tokens": 246,
        "expected_chunks": 1,
        "expected_tokens": 246
      }
    ],
    "warm_p50_ms": 1247,
    "warm_p95_ms": 1262,
    "warm_samples": 20,
    "reference_words": 300,
    "first_scan_ms": 1364,
    "first_scan_download_observed": true
  },
  "holdout_runtime": {
    "browser": "Chromium 131.0.6778.205",
    "device": "darwin 25.6.0 Apple M4 Pro",
    "engine": "ONNX Runtime Web WASM CPU, one thread per worker",
    "worker_sha256": "e6eba6365b39e878221f4d8e6b274b5ffa4092c6002a253c235a3240a21a434e",
    "signature": "80da49b4ca2df09f57c5d3efc50407d6e8ce7f1ef5a91250ee5130035dc4056a",
    "specs": {
      "live": {
        "model": "ettin",
        "version": "browser-v1",
        "upper": 0.9998051638246666,
        "lower": 0.25,
        "experimental": false
      },
      "finetuned/full": {
        "model": "finetuned",
        "scoring": "full",
        "version": "browser-v3",
        "experimental": false,
        "upper": 0.999146566314596,
        "lower": 0.25
      }
    }
  },
  "holdout": {
    "live": {
      "cohorts": {
        "asap_learner": {
          "flags": {
            "hits": 80,
            "n": 220,
            "rate": 0.36363636363636365,
            "ci95": [
              0.3029141434403334,
              0.42903916640019657
            ]
          },
          "abstention": {
            "hits": 98,
            "n": 220,
            "rate": 0.44545454545454544,
            "ci95": [
              0.3812726964449632,
              0.5115086274912487
            ]
          },
          "outside_input_limits": 0
        },
        "clean_ai": {
          "flags": {
            "hits": 215,
            "n": 300,
            "rate": 0.7166666666666667,
            "ci95": [
              0.6631845423956875,
              0.7646699712719605
            ]
          },
          "abstention": {
            "hits": 63,
            "n": 300,
            "rate": 0.21,
            "ci95": [
              0.16772109488365303,
              0.25961209451503364
            ]
          },
          "outside_input_limits": 0
        },
        "decoder": {
          "flags": {
            "hits": 18,
            "n": 150,
            "rate": 0.12,
            "ci95": [
              0.0772684043967938,
              0.18170966135395242
            ]
          },
          "abstention": {
            "hits": 71,
            "n": 150,
            "rate": 0.47333333333333333,
            "ci95": [
              0.39509757671394596,
              0.5529008840404923
            ]
          },
          "outside_input_limits": 1
        },
        "general_human": {
          "flags": {
            "hits": 10,
            "n": 300,
            "rate": 0.03333333333333333,
            "ci95": [
              0.018204749873650852,
              0.0602624514575691
            ]
          },
          "abstention": {
            "hits": 150,
            "n": 300,
            "rate": 0.5,
            "ci95": [
              0.44377849611195097,
              0.556221503888049
            ]
          },
          "outside_input_limits": 1
        },
        "seq2seq": {
          "flags": {
            "hits": 132,
            "n": 150,
            "rate": 0.88,
            "ci95": [
              0.8182903386460476,
              0.9227315956032063
            ]
          },
          "abstention": {
            "hits": 18,
            "n": 150,
            "rate": 0.12,
            "ci95": [
              0.0772684043967938,
              0.18170966135395242
            ]
          },
          "outside_input_limits": 1
        }
      },
      "paraphrase_discrimination": {
        "decoder": {
          "auroc": 0.553657605889879,
          "recall_at_1pct_fpr": 0.03355704697986577,
          "recall_at_5pct_fpr": 0.12751677852348994
        },
        "seq2seq": {
          "auroc": 0.9725258692285246,
          "recall_at_1pct_fpr": 0.825503355704698,
          "recall_at_5pct_fpr": 0.8859060402684564
        }
      }
    },
    "finetuned/full": {
      "cohorts": {
        "asap_learner": {
          "flags": {
            "hits": 0,
            "n": 220,
            "rate": 0.0,
            "ci95": [
              0,
              0.017162136081943655
            ]
          },
          "abstention": {
            "hits": 3,
            "n": 220,
            "rate": 0.013636363636363636,
            "ci95": [
              0.004648152891035449,
              0.039318652206855195
            ]
          },
          "outside_input_limits": 0
        },
        "clean_ai": {
          "flags": {
            "hits": 277,
            "n": 300,
            "rate": 0.9233333333333333,
            "ci95": [
              0.8875891334382622,
              0.9483727624969882
            ]
          },
          "abstention": {
            "hits": 23,
            "n": 300,
            "rate": 0.07666666666666666,
            "ci95": [
              0.05162723750301175,
              0.11241086656173778
            ]
          },
          "outside_input_limits": 0
        },
        "decoder": {
          "flags": {
            "hits": 89,
            "n": 150,
            "rate": 0.5933333333333334,
            "ci95": [
              0.51334517440468,
              0.6686602129547861
            ]
          },
          "abstention": {
            "hits": 59,
            "n": 150,
            "rate": 0.3933333333333333,
            "ci95": [
              0.3187585669410804,
              0.47323527607667293
            ]
          },
          "outside_input_limits": 1
        },
        "general_human": {
          "flags": {
            "hits": 1,
            "n": 300,
            "rate": 0.0033333333333333335,
            "ci95": [
              0.0005886425761407379,
              0.018637164554943353
            ]
          },
          "abstention": {
            "hits": 39,
            "n": 300,
            "rate": 0.13,
            "ci95": [
              0.09657483743516235,
              0.17278130076316203
            ]
          },
          "outside_input_limits": 1
        },
        "seq2seq": {
          "flags": {
            "hits": 110,
            "n": 150,
            "rate": 0.7333333333333333,
            "ci95": [
              0.6573840257093969,
              0.7976294426892675
            ]
          },
          "abstention": {
            "hits": 39,
            "n": 150,
            "rate": 0.26,
            "ci95": [
              0.19642034813317322,
              0.33556579865677183
            ]
          },
          "outside_input_limits": 1
        }
      },
      "paraphrase_discrimination": {
        "decoder": {
          "auroc": 0.9850957329801799,
          "recall_at_1pct_fpr": 0.7449664429530202,
          "recall_at_5pct_fpr": 0.9261744966442953
        },
        "seq2seq": {
          "auroc": 0.9826715449709322,
          "recall_at_1pct_fpr": 0.7651006711409396,
          "recall_at_5pct_fpr": 0.8523489932885906
        }
      }
    }
  },
  "paraphrase_gain": {
    "delta": 0.16333333333333333,
    "by_method": {
      "decoder": 0.47333333333333333,
      "seq2seq": -0.14666666666666667
    },
    "ci95": [
      0.10662015139065152,
      0.22040486624644678
    ],
    "bootstrap_source_groups": 300,
    "bootstrap_samples": 10000
  },
  "clean_ai": {
    "live": {
      "hits": 215,
      "n": 300,
      "rate": 0.7166666666666667,
      "ci95": [
        0.6631845423956875,
        0.7646699712719605
      ]
    },
    "candidate": {
      "hits": 277,
      "n": 300,
      "rate": 0.9233333333333333,
      "ci95": [
        0.8875891334382622,
        0.9483727624969882
      ]
    },
    "delta": 0.20666666666666667
  }
}