diff --git a/benchmark_results/20260302_203903/benchmark_results.csv b/benchmark_results/20260302_203903/benchmark_results.csv deleted file mode 100644 index 599665a..0000000 --- a/benchmark_results/20260302_203903/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,114.48470037430525,144001,740000,23.746039997786283,6064211.127978578,29.719439292042605, -crayon:cpu:lite,code,OK,39.47019996121526,556000,1048000,66.99784002266824,8298775.002475921,14.917655304344034, -crayon:cpu:lite,unicode,OK,34.600399900227785,474000,660000,48.36849998682737,9799766.379546372,13.013119054747243, -crayon:cpu:lite,mixed,OK,27.651799842715263,640000,1397500,100.7761999964714,6350705.821636548,13.224946537222081, -crayon:cpu:standard,english,OK,195.42230013757944,136001,740000,19.34308996424079,7030986.272173811,36.48429467294391, -crayon:cpu:standard,code,OK,130.2620000205934,312000,1048000,32.05965985544026,9731856.21453361,31.174712648242632, -crayon:cpu:standard,unicode,OK,88.01209973171353,216001,660000,16.120909992605448,13398809.378569707,39.044014830232165, -crayon:cpu:standard,mixed,OK,82.262699957937,372501,1397500,44.054059917107224,8455543.04645028,30.252827087570935, -tiktoken:p50k_base,english,OK,218.41720025986433,144001,740000,275.8389800786972,522047.3189065459,2.5584454885211745, -tiktoken:p50k_base,code,OK,0.012800097465515137,420000,1048000,797.0347099471837,526953.2113950617,1.253961303215991, -tiktoken:p50k_base,unicode,OK,0.008200295269489288,378000,660000,650.6462200079113,580960.8791631862,0.9673844701971399, -tiktoken:p50k_base,mixed,OK,0.0065998174250125885,517500,1397500,805.55115994066,642417.2985339889,1.6544695401790628, -tiktoken:cl100k_base,english,OK,8563.744300045073,140001,740000,561.699420074001,249245.40598876815,1.2563997200631793, -tiktoken:cl100k_base,code,OK,0.006400048732757568,308000,1048000,1669.652880076319,184469.48085755494,0.5985978855365827, -tiktoken:cl100k_base,unicode,OK,0.009799841791391373,306000,660000,822.4047600291669,372079.5584757408,0.7653470400703932, -tiktoken:cl100k_base,mixed,OK,0.00600004568696022,410000,1397500,1689.7815798409283,242634.90908605853,0.7887172360484583, -tiktoken:o200k_base,english,OK,1381.0402997769415,140001,740000,410.21160995587707,341289.7065859708,1.7203779147463258, -tiktoken:o200k_base,code,OK,0.008499715477228165,312000,1048000,639.5134400576353,487870.9038106868,1.5628298343560627, -tiktoken:o200k_base,unicode,OK,0.005000270903110504,210000,660000,304.5519299339503,689537.5775341294,2.066724873372602, -tiktoken:o200k_base,mixed,OK,0.004500150680541992,372500,1397500,899.9680500477552,413903.5824441034,1.4808968575129013, diff --git a/benchmark_results/20260302_203903/benchmark_results.json b/benchmark_results/20260302_203903/benchmark_results.json deleted file mode 100644 index 57a160d..0000000 --- a/benchmark_results/20260302_203903/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 114.48470037430525, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 23.746039997786283, - "tokens_per_sec": 6064211.127978578, - "mb_per_sec": 29.719439292042605, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 39.47019996121526, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 66.99784002266824, - "tokens_per_sec": 8298775.002475921, - "mb_per_sec": 14.917655304344034, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 34.600399900227785, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 48.36849998682737, - "tokens_per_sec": 9799766.379546372, - "mb_per_sec": 13.013119054747243, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 27.651799842715263, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 100.7761999964714, - "tokens_per_sec": 6350705.821636548, - "mb_per_sec": 13.224946537222081, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 195.42230013757944, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 19.34308996424079, - "tokens_per_sec": 7030986.272173811, - "mb_per_sec": 36.48429467294391, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 130.2620000205934, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 32.05965985544026, - "tokens_per_sec": 9731856.21453361, - "mb_per_sec": 31.174712648242632, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 88.01209973171353, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 16.120909992605448, - "tokens_per_sec": 13398809.378569707, - "mb_per_sec": 39.044014830232165, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 82.262699957937, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 44.054059917107224, - "tokens_per_sec": 8455543.04645028, - "mb_per_sec": 30.252827087570935, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 218.41720025986433, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 275.8389800786972, - "tokens_per_sec": 522047.3189065459, - "mb_per_sec": 2.5584454885211745, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.012800097465515137, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 797.0347099471837, - "tokens_per_sec": 526953.2113950617, - "mb_per_sec": 1.253961303215991, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.008200295269489288, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 650.6462200079113, - "tokens_per_sec": 580960.8791631862, - "mb_per_sec": 0.9673844701971399, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.0065998174250125885, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 805.55115994066, - "tokens_per_sec": 642417.2985339889, - "mb_per_sec": 1.6544695401790628, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 8563.744300045073, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 561.699420074001, - "tokens_per_sec": 249245.40598876815, - "mb_per_sec": 1.2563997200631793, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.006400048732757568, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 1669.652880076319, - "tokens_per_sec": 184469.48085755494, - "mb_per_sec": 0.5985978855365827, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.009799841791391373, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 822.4047600291669, - "tokens_per_sec": 372079.5584757408, - "mb_per_sec": 0.7653470400703932, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.00600004568696022, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 1689.7815798409283, - "tokens_per_sec": 242634.90908605853, - "mb_per_sec": 0.7887172360484583, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 1381.0402997769415, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 410.21160995587707, - "tokens_per_sec": 341289.7065859708, - "mb_per_sec": 1.7203779147463258, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.008499715477228165, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 639.5134400576353, - "tokens_per_sec": 487870.9038106868, - "mb_per_sec": 1.5628298343560627, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.005000270903110504, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 304.5519299339503, - "tokens_per_sec": 689537.5775341294, - "mb_per_sec": 2.066724873372602, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.004500150680541992, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 899.9680500477552, - "tokens_per_sec": 413903.5824441034, - "mb_per_sec": 1.4808968575129013, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260302_204117/benchmark_results.csv b/benchmark_results/20260302_204117/benchmark_results.csv deleted file mode 100644 index 2d3568d..0000000 --- a/benchmark_results/20260302_204117/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,57.995499577373266,144001,740000,15.741149941459298,9148061.007965358,44.83274708424514, -crayon:cpu:lite,code,OK,21.47230040282011,556000,1048000,59.891740046441555,9283417.038290484,16.687621411880016, -crayon:cpu:lite,unicode,OK,23.56130024418235,474000,660000,23.876330070197582,19852297.174918287,26.36188421661053, -crayon:cpu:lite,mixed,OK,18.32940010353923,640000,1397500,58.61230003647506,10919209.783641336,22.73856948709304, -crayon:cpu:standard,english,OK,180.30819995328784,136001,740000,21.157420054078102,6428052.175188805,33.355626174496514, -crayon:cpu:standard,code,OK,84.9953000433743,312000,1048000,29.0361100807786,10745240.9820742,34.42095655421037, -crayon:cpu:standard,unicode,OK,80.58190019801259,216001,660000,13.059470010921359,16539798.308764668,48.19682945033375, -crayon:cpu:standard,mixed,OK,93.01549987867475,372501,1397500,33.63517001271248,11074747.053730145,39.62399644996637, -tiktoken:p50k_base,english,OK,201.42409997060895,144001,740000,164.75475002080202,874032.4632935824,4.283451579098754, -tiktoken:p50k_base,code,OK,0.009399838745594025,420000,1048000,507.72541002370417,827218.7913155488,1.9684866344331449, -tiktoken:p50k_base,unicode,OK,0.004999805241823196,378000,660000,248.08915005996823,1523645.834203671,2.5370922052656475, -tiktoken:p50k_base,mixed,OK,0.005899928510189056,517500,1397500,749.1798700299114,690755.3455478436,1.7789584457528513, -tiktoken:cl100k_base,english,OK,380.74419973418117,140001,740000,267.4057499971241,523552.6910004953,2.6391317095769815, -tiktoken:cl100k_base,code,OK,0.00509992241859436,308000,1048000,591.8267999310046,520422.5290843651,1.6887553651005096, -tiktoken:cl100k_base,unicode,OK,0.005399808287620544,306000,660000,269.0659500658512,1137267.6473002606,2.339296550433377, -tiktoken:cl100k_base,mixed,OK,0.004699919372797012,410000,1397500,616.8499699328095,664667.2934824968,2.1605899686157164, -tiktoken:o200k_base,english,OK,830.4426996037364,140001,740000,401.5089299995452,348687.1387895621,1.757666994209629, -tiktoken:o200k_base,code,OK,0.0041997991502285,312000,1048000,895.654830057174,348348.481501611,1.11588823065907, -tiktoken:o200k_base,unicode,OK,0.00509992241859436,210000,660000,331.2964300625026,633873.4165061219,1.899884791120077, -tiktoken:o200k_base,mixed,OK,0.006400048732757568,372500,1397500,833.7215200066566,446791.8736186945,1.5985671776435533, diff --git a/benchmark_results/20260302_204117/benchmark_results.json b/benchmark_results/20260302_204117/benchmark_results.json deleted file mode 100644 index 39b546b..0000000 --- a/benchmark_results/20260302_204117/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 57.995499577373266, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 15.741149941459298, - "tokens_per_sec": 9148061.007965358, - "mb_per_sec": 44.83274708424514, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 21.47230040282011, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 59.891740046441555, - "tokens_per_sec": 9283417.038290484, - "mb_per_sec": 16.687621411880016, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 23.56130024418235, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 23.876330070197582, - "tokens_per_sec": 19852297.174918287, - "mb_per_sec": 26.36188421661053, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 18.32940010353923, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 58.61230003647506, - "tokens_per_sec": 10919209.783641336, - "mb_per_sec": 22.73856948709304, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 180.30819995328784, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 21.157420054078102, - "tokens_per_sec": 6428052.175188805, - "mb_per_sec": 33.355626174496514, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 84.9953000433743, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 29.0361100807786, - "tokens_per_sec": 10745240.9820742, - "mb_per_sec": 34.42095655421037, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 80.58190019801259, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 13.059470010921359, - "tokens_per_sec": 16539798.308764668, - "mb_per_sec": 48.19682945033375, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 93.01549987867475, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 33.63517001271248, - "tokens_per_sec": 11074747.053730145, - "mb_per_sec": 39.62399644996637, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 201.42409997060895, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 164.75475002080202, - "tokens_per_sec": 874032.4632935824, - "mb_per_sec": 4.283451579098754, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.009399838745594025, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 507.72541002370417, - "tokens_per_sec": 827218.7913155488, - "mb_per_sec": 1.9684866344331449, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.004999805241823196, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 248.08915005996823, - "tokens_per_sec": 1523645.834203671, - "mb_per_sec": 2.5370922052656475, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005899928510189056, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 749.1798700299114, - "tokens_per_sec": 690755.3455478436, - "mb_per_sec": 1.7789584457528513, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 380.74419973418117, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 267.4057499971241, - "tokens_per_sec": 523552.6910004953, - "mb_per_sec": 2.6391317095769815, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.00509992241859436, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 591.8267999310046, - "tokens_per_sec": 520422.5290843651, - "mb_per_sec": 1.6887553651005096, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.005399808287620544, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 269.0659500658512, - "tokens_per_sec": 1137267.6473002606, - "mb_per_sec": 2.339296550433377, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.004699919372797012, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 616.8499699328095, - "tokens_per_sec": 664667.2934824968, - "mb_per_sec": 2.1605899686157164, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 830.4426996037364, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 401.5089299995452, - "tokens_per_sec": 348687.1387895621, - "mb_per_sec": 1.757666994209629, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.0041997991502285, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 895.654830057174, - "tokens_per_sec": 348348.481501611, - "mb_per_sec": 1.11588823065907, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.00509992241859436, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 331.2964300625026, - "tokens_per_sec": 633873.4165061219, - "mb_per_sec": 1.899884791120077, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.006400048732757568, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 833.7215200066566, - "tokens_per_sec": 446791.8736186945, - "mb_per_sec": 1.5985671776435533, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260302_204615/benchmark_results.csv b/benchmark_results/20260302_204615/benchmark_results.csv deleted file mode 100644 index a0d93db..0000000 --- a/benchmark_results/20260302_204615/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,97.10660018026829,144001,740000,34.17243007570505,4213952.583441754,20.651706436363657, -crayon:cpu:lite,code,OK,36.51330014690757,556000,1048000,53.181059984490275,10454849.906379297,18.793357708274897, -crayon:cpu:lite,unicode,OK,30.903100036084652,474000,660000,37.83726999536157,12527330.858122354,16.635054508564853, -crayon:cpu:lite,mixed,OK,32.09330001845956,640000,1397500,59.02825016528368,10842266.172687659,22.578339243428417, -crayon:cpu:standard,english,OK,90.91989975422621,136001,740000,14.632110064849257,9294694.982285257,48.23084237426391, -crayon:cpu:standard,code,OK,114.00450021028519,312000,1048000,46.48610991425812,6711682.276178244,21.499985381379492, -crayon:cpu:standard,unicode,OK,92.19190012663603,216001,660000,20.056640077382326,10769550.591057481,31.382377427110605, -crayon:cpu:standard,mixed,OK,115.81340013071895,372501,1397500,34.07650995068252,10931313.11097013,39.110808563041836, -tiktoken:p50k_base,english,OK,163.94530003890395,144001,740000,159.02490988373756,905524.8017765173,4.437788989514744, -tiktoken:p50k_base,code,OK,0.0112997367978096,420000,1048000,585.210690042004,717690.2389972646,1.707847618986614, -tiktoken:p50k_base,unicode,OK,0.00690016895532608,378000,660000,257.13081010617316,1470068.8721196738,2.4478787608852706, -tiktoken:p50k_base,mixed,OK,0.005300156772136688,517500,1397500,605.4198400583118,854778.7266934566,2.2013812052300232, -tiktoken:cl100k_base,english,OK,325.88979974389076,140001,740000,203.8289399817586,686855.3602473191,3.462310083169654, -tiktoken:cl100k_base,code,OK,0.006600283086299896,308000,1048000,1057.9422200564295,291131.2112901323,0.9447119744785688, -tiktoken:cl100k_base,unicode,OK,0.008400063961744308,306000,660000,515.0799500290304,594082.5302610857,1.2219948549592157, -tiktoken:cl100k_base,mixed,OK,0.009499955922365189,410000,1397500,772.2198298666626,530936.8966487092,1.7258814208485929, -tiktoken:o200k_base,english,OK,1069.4389999844134,140001,740000,463.4919799864292,302057.0064752774,1.5226131726406313, -tiktoken:o200k_base,code,OK,0.011799857020378113,312000,1048000,724.1476599127054,430851.3543185529,1.3801752583364444, -tiktoken:o200k_base,unicode,OK,0.004600267857313156,210000,660000,315.5560499522835,665491.9150868917,1.9946537197537584, -tiktoken:o200k_base,mixed,OK,0.005600042641162872,372500,1397500,1032.7473300043494,360688.41736771306,1.290499445950755, diff --git a/benchmark_results/20260302_204615/benchmark_results.json b/benchmark_results/20260302_204615/benchmark_results.json deleted file mode 100644 index 9d50ff7..0000000 --- a/benchmark_results/20260302_204615/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 97.10660018026829, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 34.17243007570505, - "tokens_per_sec": 4213952.583441754, - "mb_per_sec": 20.651706436363657, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 36.51330014690757, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 53.181059984490275, - "tokens_per_sec": 10454849.906379297, - "mb_per_sec": 18.793357708274897, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 30.903100036084652, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 37.83726999536157, - "tokens_per_sec": 12527330.858122354, - "mb_per_sec": 16.635054508564853, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 32.09330001845956, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 59.02825016528368, - "tokens_per_sec": 10842266.172687659, - "mb_per_sec": 22.578339243428417, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 90.91989975422621, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 14.632110064849257, - "tokens_per_sec": 9294694.982285257, - "mb_per_sec": 48.23084237426391, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 114.00450021028519, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 46.48610991425812, - "tokens_per_sec": 6711682.276178244, - "mb_per_sec": 21.499985381379492, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 92.19190012663603, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 20.056640077382326, - "tokens_per_sec": 10769550.591057481, - "mb_per_sec": 31.382377427110605, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 115.81340013071895, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 34.07650995068252, - "tokens_per_sec": 10931313.11097013, - "mb_per_sec": 39.110808563041836, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 163.94530003890395, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 159.02490988373756, - "tokens_per_sec": 905524.8017765173, - "mb_per_sec": 4.437788989514744, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.0112997367978096, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 585.210690042004, - "tokens_per_sec": 717690.2389972646, - "mb_per_sec": 1.707847618986614, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.00690016895532608, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 257.13081010617316, - "tokens_per_sec": 1470068.8721196738, - "mb_per_sec": 2.4478787608852706, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005300156772136688, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 605.4198400583118, - "tokens_per_sec": 854778.7266934566, - "mb_per_sec": 2.2013812052300232, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 325.88979974389076, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 203.8289399817586, - "tokens_per_sec": 686855.3602473191, - "mb_per_sec": 3.462310083169654, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.006600283086299896, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 1057.9422200564295, - "tokens_per_sec": 291131.2112901323, - "mb_per_sec": 0.9447119744785688, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.008400063961744308, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 515.0799500290304, - "tokens_per_sec": 594082.5302610857, - "mb_per_sec": 1.2219948549592157, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.009499955922365189, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 772.2198298666626, - "tokens_per_sec": 530936.8966487092, - "mb_per_sec": 1.7258814208485929, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 1069.4389999844134, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 463.4919799864292, - "tokens_per_sec": 302057.0064752774, - "mb_per_sec": 1.5226131726406313, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.011799857020378113, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 724.1476599127054, - "tokens_per_sec": 430851.3543185529, - "mb_per_sec": 1.3801752583364444, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.004600267857313156, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 315.5560499522835, - "tokens_per_sec": 665491.9150868917, - "mb_per_sec": 1.9946537197537584, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005600042641162872, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 1032.7473300043494, - "tokens_per_sec": 360688.41736771306, - "mb_per_sec": 1.290499445950755, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260302_204615/load_time_ms.png b/benchmark_results/20260302_204615/load_time_ms.png deleted file mode 100644 index 0ea0e75..0000000 Binary files a/benchmark_results/20260302_204615/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260302_204615/mb_per_sec.png b/benchmark_results/20260302_204615/mb_per_sec.png deleted file mode 100644 index 413a3f0..0000000 Binary files a/benchmark_results/20260302_204615/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260302_204615/tokens_per_sec.png b/benchmark_results/20260302_204615/tokens_per_sec.png deleted file mode 100644 index a807083..0000000 Binary files a/benchmark_results/20260302_204615/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260302_204615/tokens_produced.png b/benchmark_results/20260302_204615/tokens_produced.png deleted file mode 100644 index 9981dc4..0000000 Binary files a/benchmark_results/20260302_204615/tokens_produced.png and /dev/null differ diff --git a/benchmark_results/20260302_204857/benchmark_results.csv b/benchmark_results/20260302_204857/benchmark_results.csv deleted file mode 100644 index b8aff2c..0000000 --- a/benchmark_results/20260302_204857/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,57.7738001011312,144001,740000,19.389820052310824,7426629.004885394,36.39636635289554, -crayon:cpu:lite,code,OK,38.07359980419278,556000,1048000,62.34249006956816,8918475.97648984,16.031613149851093, -crayon:cpu:lite,unicode,OK,28.946600388735533,474000,660000,27.969130035489798,16947255.756562516,22.50427696640735, -crayon:cpu:lite,mixed,OK,21.315999794751406,640000,1397500,80.74895991012454,7925798.68164661,16.504978623391892, -crayon:cpu:standard,english,OK,155.36740003153682,136001,740000,23.038499942049384,5903205.518679358,30.632159034476093, -crayon:cpu:standard,code,OK,144.90989968180656,312000,1048000,42.2429499682039,7385847.821585404,23.659585430137632, -crayon:cpu:standard,unicode,OK,131.74310000613332,216001,660000,21.795690106227994,9910262.026448933,28.87841796981094, -crayon:cpu:standard,mixed,OK,105.00020021572709,372501,1397500,33.55634999461472,11100760.364574237,39.717068673786684, -tiktoken:p50k_base,english,OK,224.92519998922944,144001,740000,190.08749006316066,757551.167371154,3.7126009392103323, -tiktoken:p50k_base,code,OK,0.007799826562404633,420000,1048000,513.3073099888861,818223.2978702245,1.947080558068401, -tiktoken:p50k_base,unicode,OK,0.005499925464391708,378000,660000,300.27752998284996,1258835.4513958772,2.0961443530725523, -tiktoken:p50k_base,mixed,OK,0.005000270903110504,517500,1397500,617.437299946323,838141.7838620844,2.1585347326661313, -tiktoken:cl100k_base,english,OK,317.07939971238375,140001,740000,179.39441995695233,780408.8891593997,3.933896016999693, -tiktoken:cl100k_base,code,OK,0.004699919372797012,308000,1048000,495.8733200095594,621126.3796044973,2.015536313940993, -tiktoken:cl100k_base,unicode,OK,0.0048996880650520325,306000,660000,301.8918299116194,1013608.0863453087,2.0849356837924127, -tiktoken:cl100k_base,mixed,OK,0.004400033503770828,410000,1397500,634.219199931249,646464.187846166,2.1014183382057956, -tiktoken:o200k_base,english,OK,1596.7934001237154,140001,740000,583.154410077259,240075.3515376005,1.2101751816420767, -tiktoken:o200k_base,code,OK,0.0074999406933784485,312000,1048000,839.6005300339311,371605.29184919765,1.190388342838896, -tiktoken:o200k_base,unicode,OK,0.00490015372633934,210000,660000,295.3898400068283,710924.925498946,2.1308283616443107, -tiktoken:o200k_base,mixed,OK,0.004800036549568176,372500,1397500,701.2907000724226,531163.4675342647,1.9004385157825414, diff --git a/benchmark_results/20260302_204857/benchmark_results.json b/benchmark_results/20260302_204857/benchmark_results.json deleted file mode 100644 index 9334adf..0000000 --- a/benchmark_results/20260302_204857/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 57.7738001011312, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 19.389820052310824, - "tokens_per_sec": 7426629.004885394, - "mb_per_sec": 36.39636635289554, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 38.07359980419278, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 62.34249006956816, - "tokens_per_sec": 8918475.97648984, - "mb_per_sec": 16.031613149851093, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 28.946600388735533, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 27.969130035489798, - "tokens_per_sec": 16947255.756562516, - "mb_per_sec": 22.50427696640735, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 21.315999794751406, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 80.74895991012454, - "tokens_per_sec": 7925798.68164661, - "mb_per_sec": 16.504978623391892, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 155.36740003153682, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 23.038499942049384, - "tokens_per_sec": 5903205.518679358, - "mb_per_sec": 30.632159034476093, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 144.90989968180656, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 42.2429499682039, - "tokens_per_sec": 7385847.821585404, - "mb_per_sec": 23.659585430137632, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 131.74310000613332, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 21.795690106227994, - "tokens_per_sec": 9910262.026448933, - "mb_per_sec": 28.87841796981094, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 105.00020021572709, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 33.55634999461472, - "tokens_per_sec": 11100760.364574237, - "mb_per_sec": 39.717068673786684, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 224.92519998922944, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 190.08749006316066, - "tokens_per_sec": 757551.167371154, - "mb_per_sec": 3.7126009392103323, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.007799826562404633, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 513.3073099888861, - "tokens_per_sec": 818223.2978702245, - "mb_per_sec": 1.947080558068401, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.005499925464391708, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 300.27752998284996, - "tokens_per_sec": 1258835.4513958772, - "mb_per_sec": 2.0961443530725523, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005000270903110504, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 617.437299946323, - "tokens_per_sec": 838141.7838620844, - "mb_per_sec": 2.1585347326661313, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 317.07939971238375, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 179.39441995695233, - "tokens_per_sec": 780408.8891593997, - "mb_per_sec": 3.933896016999693, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.004699919372797012, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 495.8733200095594, - "tokens_per_sec": 621126.3796044973, - "mb_per_sec": 2.015536313940993, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.0048996880650520325, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 301.8918299116194, - "tokens_per_sec": 1013608.0863453087, - "mb_per_sec": 2.0849356837924127, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.004400033503770828, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 634.219199931249, - "tokens_per_sec": 646464.187846166, - "mb_per_sec": 2.1014183382057956, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 1596.7934001237154, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 583.154410077259, - "tokens_per_sec": 240075.3515376005, - "mb_per_sec": 1.2101751816420767, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.0074999406933784485, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 839.6005300339311, - "tokens_per_sec": 371605.29184919765, - "mb_per_sec": 1.190388342838896, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.00490015372633934, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 295.3898400068283, - "tokens_per_sec": 710924.925498946, - "mb_per_sec": 2.1308283616443107, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.004800036549568176, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 701.2907000724226, - "tokens_per_sec": 531163.4675342647, - "mb_per_sec": 1.9004385157825414, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260302_204857/load_time_ms.png b/benchmark_results/20260302_204857/load_time_ms.png deleted file mode 100644 index a96b2cb..0000000 Binary files a/benchmark_results/20260302_204857/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260302_204857/mb_per_sec.png b/benchmark_results/20260302_204857/mb_per_sec.png deleted file mode 100644 index 41f8110..0000000 Binary files a/benchmark_results/20260302_204857/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260302_204857/tokens_per_sec.png b/benchmark_results/20260302_204857/tokens_per_sec.png deleted file mode 100644 index a7144c4..0000000 Binary files a/benchmark_results/20260302_204857/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260302_204857/tokens_produced.png b/benchmark_results/20260302_204857/tokens_produced.png deleted file mode 100644 index 9981dc4..0000000 Binary files a/benchmark_results/20260302_204857/tokens_produced.png and /dev/null differ diff --git a/benchmark_results/20260316_125203/benchmark_results.csv b/benchmark_results/20260316_125203/benchmark_results.csv deleted file mode 100644 index 97d4c51..0000000 --- a/benchmark_results/20260316_125203/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,97.45750017464161,144001,740000,27.60476681093375,5216526.587102478,25.565113408641526, -crayon:cpu:lite,code,OK,31.84089995920658,556000,1048000,128.8694003596902,4314445.465317106,7.755531420214274, -crayon:cpu:lite,unicode,OK,57.07800015807152,474000,660000,181.54360043505827,2610943.0399313862,3.4670737350132197, -crayon:cpu:lite,mixed,OK,25.56310035288334,640000,1397500,213.41429992268482,2998861.839304384,6.244941682261044, -crayon:cpu:standard,english,OK,260.6979003176093,136001,740000,58.22526663541794,2335772.9016782506,12.120493986906745, -crayon:cpu:standard,code,OK,289.5730007439852,312000,1048000,169.16560009121895,1844346.6037525397,5.908120108667586, -crayon:cpu:standard,unicode,OK,142.52520073205233,216001,660000,31.140000248948734,6936448.242555555,20.212750282472268, -crayon:cpu:standard,mixed,OK,207.64579996466637,372501,1397500,66.71659989903371,5583333.09197004,19.976435537702475, -tiktoken:p50k_base,english,OK,347.60630037635565,144001,740000,507.0427001143495,284001.72207887925,1.391833456987882, -tiktoken:p50k_base,code,OK,0.01769978553056717,420000,1048000,2520.244666685661,166650.48657848613,0.39656891126690247, -tiktoken:p50k_base,unicode,OK,0.015099532902240753,378000,660000,930.4823335260153,406240.92084326665,0.6764502948088772, -tiktoken:p50k_base,mixed,OK,0.006899237632751465,517500,1397500,2307.905833236873,224229.2525749191,0.5774758389117282, -tiktoken:cl100k_base,english,OK,760.9403003007174,140001,740000,532.0695002252857,263125.3998598334,1.3263661868267467, -tiktoken:cl100k_base,code,OK,0.0064997002482414246,308000,1048000,1633.9926331614454,188495.3418694932,0.6116616827457876, -tiktoken:cl100k_base,unicode,OK,0.00690016895532608,306000,660000,908.9693001781901,336645.0329400709,0.6924601839740192, -tiktoken:cl100k_base,mixed,OK,0.006599351763725281,410000,1397500,1150.4462001224358,356383.46230911615,1.1584721276283028, -tiktoken:o200k_base,english,OK,1201.8392998725176,140001,740000,575.7089667022228,243180.16236911158,1.2258259554009134, -tiktoken:o200k_base,code,OK,0.007599592208862305,312000,1048000,1332.6717000454664,234116.17429060405,0.7499601616509542, -tiktoken:o200k_base,unicode,OK,0.0061998143792152405,210000,660000,643.3313333739837,326425.88524119346,0.9783839464604863, -tiktoken:o200k_base,mixed,OK,0.005499459803104401,372500,1397500,1549.1690002381802,240451.49363479984,0.8603063042010436, diff --git a/benchmark_results/20260316_125203/benchmark_results.json b/benchmark_results/20260316_125203/benchmark_results.json deleted file mode 100644 index 6022a88..0000000 --- a/benchmark_results/20260316_125203/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 97.45750017464161, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 27.60476681093375, - "tokens_per_sec": 5216526.587102478, - "mb_per_sec": 25.565113408641526, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 31.84089995920658, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 128.8694003596902, - "tokens_per_sec": 4314445.465317106, - "mb_per_sec": 7.755531420214274, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 57.07800015807152, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 181.54360043505827, - "tokens_per_sec": 2610943.0399313862, - "mb_per_sec": 3.4670737350132197, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 25.56310035288334, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 213.41429992268482, - "tokens_per_sec": 2998861.839304384, - "mb_per_sec": 6.244941682261044, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 260.6979003176093, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 58.22526663541794, - "tokens_per_sec": 2335772.9016782506, - "mb_per_sec": 12.120493986906745, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 289.5730007439852, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 169.16560009121895, - "tokens_per_sec": 1844346.6037525397, - "mb_per_sec": 5.908120108667586, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 142.52520073205233, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 31.140000248948734, - "tokens_per_sec": 6936448.242555555, - "mb_per_sec": 20.212750282472268, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 207.64579996466637, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 66.71659989903371, - "tokens_per_sec": 5583333.09197004, - "mb_per_sec": 19.976435537702475, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 347.60630037635565, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 507.0427001143495, - "tokens_per_sec": 284001.72207887925, - "mb_per_sec": 1.391833456987882, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.01769978553056717, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 2520.244666685661, - "tokens_per_sec": 166650.48657848613, - "mb_per_sec": 0.39656891126690247, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.015099532902240753, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 930.4823335260153, - "tokens_per_sec": 406240.92084326665, - "mb_per_sec": 0.6764502948088772, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.006899237632751465, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 2307.905833236873, - "tokens_per_sec": 224229.2525749191, - "mb_per_sec": 0.5774758389117282, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 760.9403003007174, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 532.0695002252857, - "tokens_per_sec": 263125.3998598334, - "mb_per_sec": 1.3263661868267467, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.0064997002482414246, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 1633.9926331614454, - "tokens_per_sec": 188495.3418694932, - "mb_per_sec": 0.6116616827457876, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.00690016895532608, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 908.9693001781901, - "tokens_per_sec": 336645.0329400709, - "mb_per_sec": 0.6924601839740192, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.006599351763725281, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 1150.4462001224358, - "tokens_per_sec": 356383.46230911615, - "mb_per_sec": 1.1584721276283028, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 1201.8392998725176, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 575.7089667022228, - "tokens_per_sec": 243180.16236911158, - "mb_per_sec": 1.2258259554009134, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.007599592208862305, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 1332.6717000454664, - "tokens_per_sec": 234116.17429060405, - "mb_per_sec": 0.7499601616509542, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.0061998143792152405, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 643.3313333739837, - "tokens_per_sec": 326425.88524119346, - "mb_per_sec": 0.9783839464604863, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005499459803104401, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 1549.1690002381802, - "tokens_per_sec": 240451.49363479984, - "mb_per_sec": 0.8603063042010436, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_125203/load_time_ms.png b/benchmark_results/20260316_125203/load_time_ms.png deleted file mode 100644 index 4424dbe..0000000 Binary files a/benchmark_results/20260316_125203/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260316_125203/mb_per_sec.png b/benchmark_results/20260316_125203/mb_per_sec.png deleted file mode 100644 index ee925c3..0000000 Binary files a/benchmark_results/20260316_125203/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_125203/tokens_per_sec.png b/benchmark_results/20260316_125203/tokens_per_sec.png deleted file mode 100644 index 08ac95c..0000000 Binary files a/benchmark_results/20260316_125203/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_125203/tokens_produced.png b/benchmark_results/20260316_125203/tokens_produced.png deleted file mode 100644 index ea990ae..0000000 Binary files a/benchmark_results/20260316_125203/tokens_produced.png and /dev/null differ diff --git a/benchmark_results/20260316_130952/benchmark_results.csv b/benchmark_results/20260316_130952/benchmark_results.csv deleted file mode 100644 index fed0f87..0000000 --- a/benchmark_results/20260316_130952/benchmark_results.csv +++ /dev/null @@ -1,401 +0,0 @@ -impl,case,status,cold_load_time_ms,warm_load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:lite,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,english,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,code,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,unicode,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" -crayon:cpu:standard,mixed,FAIL,0.0,0.0,0,0,0.0,0.0,0.0,"Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try: - pip install --force-reinstall xerv-crayon -Or for development: - pip install -e . -" diff --git a/benchmark_results/20260316_130952/benchmark_results.json b/benchmark_results/20260316_130952/benchmark_results.json deleted file mode 100644 index 40d8ced..0000000 --- a/benchmark_results/20260316_130952/benchmark_results.json +++ /dev/null @@ -1,1042 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "FAIL", - "cold_load_time_ms": 0.0, - "warm_load_time_ms": 0.0, - "tokens_produced": 0, - "bytes_processed": 0, - "avg_time_ms": 0.0, - "tokens_per_sec": 0.0, - "mb_per_sec": 0.0, - "notes": "Critical Crayon Error: 'crayon_cpu' extension not found. The package may not be installed correctly. Try:\n pip install --force-reinstall xerv-crayon\nOr for development:\n pip install -e .\n" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_130952/benchmark_summary.csv b/benchmark_results/20260316_130952/benchmark_summary.csv deleted file mode 100644 index 2031b10..0000000 --- a/benchmark_results/20260316_130952/benchmark_summary.csv +++ /dev/null @@ -1 +0,0 @@ -impl,case,n,tokens_per_sec_mean,tokens_per_sec_std,cold_load_time_ms_mean,cold_load_time_ms_std,warm_load_time_ms_mean,warm_load_time_ms_std,mb_per_sec_mean,mb_per_sec_std,tokens_produced_mean,tokens_produced_std diff --git a/benchmark_results/20260316_130952/benchmark_summary.json b/benchmark_results/20260316_130952/benchmark_summary.json deleted file mode 100644 index 0637a08..0000000 --- a/benchmark_results/20260316_130952/benchmark_summary.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/benchmark_results/20260316_130952/metadata.json b/benchmark_results/20260316_130952/metadata.json deleted file mode 100644 index 77611c1..0000000 --- a/benchmark_results/20260316_130952/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "timestamp": "2026-03-16T13:09:52", - "cwd": "/app", - "device_arg": "cpu", - "platform": "Linux-6.8.0-x86_64-with-glibc2.39", - "python": "3.12.13 (main, Mar 6 2026, 16:37:31) [GCC 13.3.0]", - "processor": "x86_64", - "cpu_count_logical": 4, - "ram_total_bytes": null, - "nvidia_smi": null, - "rocm_smi": null, - "versions": { - "tiktoken": null, - "transformers": null, - "matplotlib": null - }, - "backends": { - "torch": null - } -} \ No newline at end of file diff --git a/benchmark_results/20260316_131110/benchmark_results.csv b/benchmark_results/20260316_131110/benchmark_results.csv deleted file mode 100644 index d0ccbd9..0000000 --- a/benchmark_results/20260316_131110/benchmark_results.csv +++ /dev/null @@ -1,201 +0,0 @@ -impl,case,status,cold_load_time_ms,warm_load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,13.420045999993135,8.633935999995401,144001,740000,9.07103429999836,15874815.95125553,77.7991760146638, -crayon:cpu:lite,code,OK,8.61052400000517,8.313041999997495,556000,1048000,19.86401780000051,27990309.191123746,50.31462887602348, -crayon:cpu:lite,unicode,OK,12.350933999982772,8.243020000008983,474000,660000,12.465735299994662,38024231.109752744,50.49241249558656, -crayon:cpu:lite,mixed,OK,12.343785000012986,8.08141199999568,640000,1397500,26.806486600000312,23874818.418016504,49.717811851468774, -crayon:cpu:standard,english,OK,57.44562099999939,56.62920999998278,136001,740000,8.589664400003016,15833098.205786971,82.15908809433546, -crayon:cpu:standard,code,OK,59.483790999991015,62.2261299999991,312000,1048000,20.4121071000003,15285046.196920818,48.96362137908513, -crayon:cpu:standard,unicode,OK,56.097327999992785,61.74585100001195,216001,660000,8.520422300000519,25350973.507497024,73.87251789480982, -crayon:cpu:standard,mixed,OK,56.56124600000112,62.25921600000106,372501,1397500,24.91756019999798,14949336.813482655,53.486771838032624, -tiktoken:p50k_base,english,OK,871.5962149999825,0.008374000003641413,144001,740000,68.25013059999776,2109900.7244977304,10.340185255860128, -tiktoken:p50k_base,code,OK,0.007124999996221959,0.0005149999822151585,420000,1048000,219.8429060000052,1910455.09560354,4.546203931609813, -tiktoken:p50k_base,unicode,OK,0.0070859999823369435,0.0004989999808913126,378000,660000,110.32828590000179,3426138.6091197655,5.705019738987124, -tiktoken:p50k_base,mixed,OK,0.005673000003980633,0.0004380000007131457,517500,1397500,240.371629699996,2152916.30150315,5.544580526583486, -tiktoken:cl100k_base,english,OK,796.1088989999894,0.005170000008547504,140001,740000,84.89410629999554,1649125.0818433713,8.312932721698987, -tiktoken:cl100k_base,code,OK,0.005845000004001122,0.000515000010636868,308000,1048000,218.6228982000017,1408818.5754368412,4.5715736632465065, -tiktoken:cl100k_base,unicode,OK,0.006498000004739879,0.0004240000066602079,306000,660000,128.00079199999175,2390610.2080994914,4.917352767849792, -tiktoken:cl100k_base,mixed,OK,0.007242000009455296,0.0004219999993892998,410000,1397500,247.372390299995,1657420.2137222437,5.38766616420475, -tiktoken:o200k_base,english,OK,1123.7693420000028,0.007920000001604421,140001,740000,148.18562679999445,944767.7418064222,4.76239841461231, -tiktoken:o200k_base,code,OK,0.005945999987488904,0.0004309999894758221,312000,1048000,527.5963346000026,591361.1970722711,1.8943472841817297, -tiktoken:o200k_base,unicode,OK,0.006991999981664776,0.0004199999921183917,210000,660000,250.5198507000017,838256.9262005335,2.51247574621088, -tiktoken:o200k_base,mixed,OK,0.005476999945130956,0.00038700000004610047,372500,1397500,399.6432679999941,932081.2580283611,3.334873783430662, -crayon:cpu:lite,english,OK,12.255995000032271,8.353256999953373,144001,740000,7.629351100001713,18874606.51797342,92.5005265704008, -crayon:cpu:lite,code,OK,8.631075999971927,8.182028000021546,556000,1048000,17.14357649999556,32431972.406699616,58.29884351109635, -crayon:cpu:lite,unicode,OK,8.595698000021912,8.26907499998697,474000,660000,11.279014600006576,42024947.81766872,55.804968000285946, -crayon:cpu:lite,mixed,OK,8.619954999971924,8.18737700001293,640000,1397500,27.430875499999274,23331373.43720644,48.586121765518186, -crayon:cpu:standard,english,OK,57.70983200000046,57.04072999998289,136001,740000,7.748942700010275,17550910.526131477,91.07294007215839, -crayon:cpu:standard,code,OK,55.32362400003876,56.88838099996474,312000,1048000,18.391508200005546,16964350.971493784,54.34305184353769, -crayon:cpu:standard,unicode,OK,53.574158000003536,56.808552000006785,216001,660000,7.41275420001557,29139101.900822006,84.91109132241333, -crayon:cpu:standard,mixed,OK,53.982016999952975,56.91404300000613,372501,1397500,22.914766900004224,16255936.690323973,58.16161530220448, -tiktoken:p50k_base,english,OK,0.007771000014145102,0.0005619999683403876,144001,740000,68.49571729999866,2102335.8200528375,10.3031112302935, -tiktoken:p50k_base,code,OK,0.006550999955834413,0.000489000001380191,420000,1048000,219.48089210000603,1913606.2186617495,4.553702484216039, -tiktoken:p50k_base,unicode,OK,0.0067070000113744754,0.0005200000146032835,378000,660000,107.88554559999852,3503713.1053819796,5.834192572578821, -tiktoken:p50k_base,mixed,OK,0.007401000004847447,0.00043800002913485514,517500,1397500,241.58617579999486,2142092.767876046,5.516705799760264, -tiktoken:cl100k_base,english,OK,0.010244000009151932,0.00043000000005122274,140001,740000,78.11836440000661,1792165.0187543887,9.03397043142093, -tiktoken:cl100k_base,code,OK,0.008102000037979451,0.0004949999947712058,308000,1048000,216.04720779998843,1425614.351309457,4.626075447913303, -tiktoken:cl100k_base,unicode,OK,0.006573999996817292,0.00040900005160438013,306000,660000,123.78520560000084,2472024.0073664985,5.084816442944299, -tiktoken:cl100k_base,mixed,OK,0.006136000024525856,0.0004350000040176383,410000,1397500,243.61621470000046,1682975.0043727253,5.470735430393878, -tiktoken:o200k_base,english,OK,0.006606999988889584,0.0008569999749852286,140001,740000,147.9927857000007,945998.8156706436,4.76860402892343, -tiktoken:o200k_base,code,OK,0.006286000029831484,0.0004669999498219113,312000,1048000,348.46320080000623,895359.9670889392,2.8681670870817877, -tiktoken:o200k_base,unicode,OK,0.005886999986159935,0.0004969999736204045,210000,660000,167.48856810000348,1253816.9164752422,3.758017970828374, -tiktoken:o200k_base,mixed,OK,0.012467000033211662,0.00045899997758169775,372500,1397500,386.2139582999987,964491.2929600899,3.450832960683645, -crayon:cpu:lite,english,OK,14.46436700001641,8.621505999997225,144001,740000,7.970677099996237,18066344.70239272,88.53940327616058, -crayon:cpu:lite,code,OK,8.487282999965373,8.221886000001177,556000,1048000,17.167496500002244,32386783.943711728,58.217613942348514, -crayon:cpu:lite,unicode,OK,8.32785199997943,7.899138000027506,474000,660000,11.294528399997716,41967223.70454138,55.72831609580549, -crayon:cpu:lite,mixed,OK,8.416824999983419,8.190843000022596,640000,1397500,25.39328400000045,25203514.441062003,52.48473797944806, -crayon:cpu:standard,english,OK,59.122182999999495,56.704432000003635,136001,740000,7.6757321999991746,17718309.661717307,91.94158625553675, -crayon:cpu:standard,code,OK,57.78209900000775,57.2303270000134,312000,1048000,17.449783000000707,17879878.506224826,57.27582306288333, -crayon:cpu:standard,unicode,OK,52.829341000006025,55.68740900002922,216001,660000,7.266771999991306,29724477.38834498,86.61687043832916, -crayon:cpu:standard,mixed,OK,54.2102670000304,56.79054400002315,372501,1397500,22.231868899996243,16755271.528254783,59.94817004241868, -tiktoken:p50k_base,english,OK,0.005590999990090495,0.0004799999828719592,144001,740000,67.79733839999267,2123991.9353532554,10.409243353728785, -tiktoken:p50k_base,code,OK,0.00664499998492829,0.00041600003441999434,420000,1048000,215.4200641999921,1949679.1144304904,4.639543151680978, -tiktoken:p50k_base,unicode,OK,0.006171000052290765,0.0005260000079942984,378000,660000,107.30631869999456,3522625.7370435647,5.865684858576338, -tiktoken:p50k_base,mixed,OK,0.005192000003262365,0.0004579999881570984,517500,1397500,236.72034209999424,2186123.91064136,5.630102784384945, -tiktoken:cl100k_base,english,OK,0.007005999975717714,0.0004910000370728085,140001,740000,77.01460709998855,1817849.9543370495,9.16344341307079, -tiktoken:cl100k_base,code,OK,0.006001999963700655,0.0004839999974137754,308000,1048000,212.1594686999913,1451738.1754737238,4.710846467131028, -tiktoken:cl100k_base,unicode,OK,0.005649999991419463,0.00044099999740865314,306000,660000,121.5911033999987,2516631.492300474,5.176571568377853, -tiktoken:cl100k_base,mixed,OK,0.0057810000271274475,0.0004139999987273768,410000,1397500,240.63990579999768,1703790.5605762748,5.53839918091319, -tiktoken:o200k_base,english,OK,0.007540000012795645,0.0004569999987324991,140001,740000,147.87910310000711,946726.0557113377,4.772269910667257, -tiktoken:o200k_base,code,OK,0.0067850000391445064,0.0004030000013699464,312000,1048000,344.18638970000757,906485.5826284671,2.9038065231599366, -tiktoken:o200k_base,unicode,OK,0.005466999994041544,0.0005240000291450997,210000,660000,165.9669846999975,1265311.8954929302,3.7924714362068808, -tiktoken:o200k_base,mixed,OK,0.005138000005899812,0.00040200001194534707,372500,1397500,386.74107549999803,963176.7184760851,3.44612957249156, -crayon:cpu:lite,english,OK,11.859816000026058,8.044323999968128,144001,740000,7.576637799991204,19005923.70934865,93.14408485270924, -crayon:cpu:lite,code,OK,8.248379999997724,7.75930600002539,556000,1048000,16.88808420000214,32922621.264520314,59.18082073475352, -crayon:cpu:lite,unicode,OK,8.604470999955538,8.002074999978959,474000,660000,11.156769700005498,42485415.82782393,56.4164239069858, -crayon:cpu:lite,mixed,OK,8.456352999985484,8.093239999993784,640000,1397500,23.72167619999459,26979543.71370038,56.18320754154963, -crayon:cpu:standard,english,OK,57.44187900000952,57.31347900001538,136001,740000,7.547879199995577,18018438.874866955,93.49897838071368, -crayon:cpu:standard,code,OK,55.624494000028335,56.39792900001339,312000,1048000,16.869116499987058,18495337.322511192,59.24736387910515, -crayon:cpu:standard,unicode,OK,52.97563800002081,56.39938499996333,216001,660000,7.500860099992224,28796830.912794113,83.91371661881516, -crayon:cpu:standard,mixed,OK,53.65096499997435,56.49196399997436,372501,1397500,21.90999329999954,17001420.078024752,60.828857358790806, -tiktoken:p50k_base,english,OK,0.006266999946547003,0.0005079999709778349,144001,740000,67.73963460000232,2125801.2513695355,10.41811043575663, -tiktoken:p50k_base,code,OK,0.006395000013981189,0.00042300001723560854,420000,1048000,215.189668499994,1951766.5644808207,4.6445105406804315, -tiktoken:p50k_base,unicode,OK,0.006347999999434251,0.00045899997758169775,378000,660000,107.80626470000243,3506289.7416201034,5.838483047155522, -tiktoken:p50k_base,mixed,OK,0.005572000020492851,0.0004800000397153781,517500,1397500,240.88481109999975,2148329.7250533886,5.53276834305862, -tiktoken:cl100k_base,english,OK,0.0059879999980694265,0.0004139999987273768,140001,740000,77.52655099998833,1805845.8449934279,9.102932931206126, -tiktoken:cl100k_base,code,OK,0.03849000000855085,0.0005480000027091592,308000,1048000,212.68630399998187,1448142.142711861,4.699177449591842, -tiktoken:cl100k_base,unicode,OK,0.006583000015325524,0.00044500001195046934,306000,660000,122.85836879999579,2490672.820979294,5.123176019476392, -tiktoken:cl100k_base,mixed,OK,0.0062469999875247595,0.0005079999709778349,410000,1397500,241.95004649999987,1694564.667091313,5.508409179734277, -tiktoken:o200k_base,english,OK,0.006681000002117798,0.00043800002913485514,140001,740000,149.0840187000117,939074.4978622515,4.733699831104497, -tiktoken:o200k_base,code,OK,0.00601800002186792,0.0004799999828719592,312000,1048000,344.87705079999387,904670.2274804002,2.8979912733403825, -tiktoken:o200k_base,unicode,OK,0.006578000011359109,0.0004529999841906829,210000,660000,166.56413929999871,1260775.5840035228,3.778874921536786, -tiktoken:o200k_base,mixed,OK,0.005728999951770675,0.0005079999709778349,372500,1397500,386.02468130000034,964964.2057744759,3.452524985421789, -crayon:cpu:lite,english,OK,11.945968000020457,8.068296999965696,144001,740000,7.684993499987058,18737946.88313562,91.8307860822281, -crayon:cpu:lite,code,OK,8.50575999999137,8.087123999985124,556000,1048000,16.832955599988964,33030444.16040428,59.37464027971447, -crayon:cpu:lite,unicode,OK,8.485736000011457,7.785606999959782,474000,660000,11.008489499999996,43057678.34905962,57.176331850807074, -crayon:cpu:lite,mixed,OK,8.317911999995431,7.874027000013939,640000,1397500,23.156396000007362,27638152.32732229,57.55471866940394, -crayon:cpu:standard,english,OK,56.3960169999973,55.77992599995696,136001,740000,7.532129799994891,18056114.752575327,93.69448122642598, -crayon:cpu:standard,code,OK,55.15872899997021,55.88644499999873,312000,1048000,16.60530089999952,18789180.74890224,60.18865238351561, -crayon:cpu:standard,unicode,OK,52.26000599998315,55.77487499999734,216001,660000,7.332337400015376,29458682.575019944,85.8423466474422, -crayon:cpu:standard,mixed,OK,52.8641419999758,56.416287999979886,372501,1397500,22.371619800003373,16650604.798850723,59.57368617437051, -tiktoken:p50k_base,english,OK,0.006202999998095038,0.0004969999736204045,144001,740000,84.53547129999492,1703438.778840861,8.348199676277991, -tiktoken:p50k_base,code,OK,0.013772000045264576,0.0006299999881775875,420000,1048000,331.9959505999975,1265075.671076583,3.010430343465031, -tiktoken:p50k_base,unicode,OK,0.00607499998750427,0.0007910000476840651,378000,660000,107.05100760001187,3531027.0166944047,5.879674212688636, -tiktoken:p50k_base,mixed,OK,0.005955999995421735,0.00043699998286683694,517500,1397500,236.48260150000056,2188321.663908957,5.635762837198538, -tiktoken:cl100k_base,english,OK,0.006967000047097827,0.0005080000278212538,140001,740000,76.65469670000107,1826385.1535139927,9.20646776416787, -tiktoken:cl100k_base,code,OK,0.005427000019153638,0.0005530000066755747,308000,1048000,209.65246490000027,1469097.9194874212,4.767178311356685, -tiktoken:cl100k_base,unicode,OK,0.006586000040442741,0.0005190000251786842,306000,660000,121.59806469999808,2516487.4190633795,5.176275217710228, -tiktoken:cl100k_base,mixed,OK,0.006361999965065479,0.0004229999603921897,410000,1397500,240.8612956000013,1702224.5063436327,5.533308512095071, -tiktoken:o200k_base,english,OK,0.007159999995565158,0.0005079999709778349,140001,740000,146.5338551000059,955417.4351343765,4.816081537327934, -tiktoken:o200k_base,code,OK,0.005568999995375634,0.00043200003574384027,312000,1048000,341.9993831000113,912282.3473303209,2.9223756912493593, -tiktoken:o200k_base,unicode,OK,0.0060189999544491,0.0004590000344251166,210000,660000,166.09841920000008,1264310.6479366175,3.789470434816304, -tiktoken:o200k_base,mixed,OK,0.005809000015233323,0.0004269999749340059,372500,1397500,385.07791709998287,967336.6959219397,3.4610134676502167, -crayon:cpu:lite,english,OK,12.146585999971649,8.18777499995349,144001,740000,7.5930362000008245,18964877.317453638,92.94292501075503, -crayon:cpu:lite,code,OK,8.266219999995883,7.896754000000783,556000,1048000,16.99046889999636,32724229.288346432,58.82419664085694, -crayon:cpu:lite,unicode,OK,8.481736000021556,7.952439000007416,474000,660000,11.347498700001779,41771320.05310789,55.46817545156681, -crayon:cpu:lite,mixed,OK,8.46451799998249,8.242848000008962,640000,1397500,24.098211299997274,26557987.72915865,55.30534364506486, -crayon:cpu:standard,english,OK,57.78302799996027,57.101206000027105,136001,740000,7.574477499991872,17955165.88439875,93.17065027143883, -crayon:cpu:standard,code,OK,55.734033000021554,57.81873300003326,312000,1048000,16.75580560000185,18620411.781333007,59.6480233450333, -crayon:cpu:standard,unicode,OK,53.034622000041054,56.92579700001943,216001,660000,7.35246310000548,29378046.113531534,85.60737269496204, -crayon:cpu:standard,mixed,OK,53.62937799998235,56.189039999992474,372501,1397500,22.484254099992995,16567194.016906083,59.27525330618593, -tiktoken:p50k_base,english,OK,0.00639499995713777,0.00045899997758169775,144001,740000,68.15755910000121,2112766.388666014,10.354229280793191, -tiktoken:p50k_base,code,OK,0.0067010000179834606,0.0005269999974188977,420000,1048000,214.87836619999712,1954594.161466617,4.651239216252769, -tiktoken:p50k_base,unicode,OK,0.01659599996628458,0.0004880000119555916,378000,660000,107.26061450000088,3524126.7427196857,5.868184251621268, -tiktoken:p50k_base,mixed,OK,0.006431000031170697,0.0007040000014058023,517500,1397500,236.91750270000966,2184304.638122369,5.625417463838901, -tiktoken:cl100k_base,english,OK,0.006840999958512839,0.0005940000278314983,140001,740000,77.5616793999859,1805027.9607538444,9.09881012891984, -tiktoken:cl100k_base,code,OK,0.00674799997568698,0.0004689999855145288,308000,1048000,209.97482439998976,1466842.5173356878,4.7598595995958775, -tiktoken:cl100k_base,unicode,OK,0.006465000012667588,0.000392000004012516,306000,660000,121.70541190000108,2514267.8145769234,5.171709614238768, -tiktoken:cl100k_base,mixed,OK,0.006374999998115527,0.0004489999696488667,410000,1397500,239.08225989999323,1714890.934072234,5.574482430169517, -tiktoken:o200k_base,english,OK,0.006678999966425181,0.0004799999828719592,140001,740000,146.80623299998956,953644.7951771227,4.807145989098944, -tiktoken:o200k_base,code,OK,0.0051750000125139195,0.00044500001195046934,312000,1048000,342.70248529999776,910410.6721807951,2.9163800277632728, -tiktoken:o200k_base,unicode,OK,0.0055859999861240794,0.00046999997493912815,210000,660000,165.92932179999593,1265599.0979889918,3.793332257374058, -tiktoken:o200k_base,mixed,OK,0.007171999982347188,0.00043299996832502075,372500,1397500,384.91586170000005,967743.9593027817,3.4624706066711157, -crayon:cpu:lite,english,OK,11.53909399999975,7.790105999958996,144001,740000,7.39537980001046,19471751.809122276,95.42701162415308, -crayon:cpu:lite,code,OK,8.380036000005475,7.807446000015261,556000,1048000,16.601938900004143,33490064.22375529,60.20084097487556, -crayon:cpu:lite,unicode,OK,8.321920000014416,7.639664000009816,474000,660000,10.848814999997103,43691407.77127517,58.017861750642176, -crayon:cpu:lite,mixed,OK,8.218655000007402,7.681361000038578,640000,1397500,23.521902400000272,27208683.596952286,56.660376976044205, -crayon:cpu:standard,english,OK,55.9084590000225,55.01668699997708,136001,740000,7.247118899988436,18766216.185609568,97.37924875797898, -crayon:cpu:standard,code,OK,54.783281999959854,54.293733000008615,312000,1048000,16.26572840000904,19181434.260258924,61.4451845632191, -crayon:cpu:standard,unicode,OK,51.37580899997829,54.80919600000789,216001,660000,7.13365659998999,30279141.835941903,88.23315784909077, -crayon:cpu:standard,mixed,OK,52.157340000007935,55.81748999998126,372501,1397500,22.280753100005768,16718510.291283807,59.81664314467851, -tiktoken:p50k_base,english,OK,0.004771000021719374,0.0004709999643637275,144001,740000,67.3320732000036,2138668.7377389753,10.481171314068304, -tiktoken:p50k_base,code,OK,0.004755999952976708,0.00044099999740865314,420000,1048000,214.3942374999881,1959007.876785976,4.661742289569726, -tiktoken:p50k_base,unicode,OK,0.005805999990116106,0.0005059999921286362,378000,660000,106.82626730001061,3538455.564851159,5.892043827202623, -tiktoken:p50k_base,mixed,OK,0.005986000019220228,0.0004770000145981612,517500,1397500,234.7759601000064,2204229.0862299656,5.67673051623354, -tiktoken:cl100k_base,english,OK,0.006902999984959024,0.000596000006680697,140001,740000,76.99923400001012,1818212.8928708772,9.165272918695948, -tiktoken:cl100k_base,code,OK,0.005643000008603849,0.00046999997493912815,308000,1048000,209.2496414999914,1471926.0582341913,4.776355536044209, -tiktoken:cl100k_base,unicode,OK,0.00608600004170512,0.00044500001195046934,306000,660000,121.43914309998536,2519780.625823906,5.183049161586195, -tiktoken:cl100k_base,mixed,OK,0.005396000005930546,0.000392000004012516,410000,1397500,239.10487489999355,1714728.736381824,5.573955184875113, -tiktoken:o200k_base,english,OK,0.0062280000179271156,0.00048499998683837475,140001,740000,146.86210019999066,953282.0231315806,4.80531732271026, -tiktoken:o200k_base,code,OK,0.012413000035849109,0.0005310000119607139,312000,1048000,341.3278896999884,914077.077833381,2.9281248727498403, -tiktoken:o200k_base,unicode,OK,0.00512899998739158,0.0004539999736152822,210000,660000,165.44453200000362,1269307.588841893,3.804447576593896, -tiktoken:o200k_base,mixed,OK,0.006564999978309061,0.00038000001723048626,372500,1397500,383.76850569999306,970637.2317357326,3.4728223848040445, -crayon:cpu:lite,english,OK,11.782717000016873,7.8747029999703955,144001,740000,7.522683499996674,19142238.27176348,93.8121342126034, -crayon:cpu:lite,code,OK,8.217624999986128,7.702755999957844,556000,1048000,16.541529200003424,33612370.00989515,60.420694574812536, -crayon:cpu:lite,unicode,OK,8.531998000023577,7.964495000010174,474000,660000,11.138282199999594,42555933.804587685,56.510064795103496, -crayon:cpu:lite,mixed,OK,8.412228999986837,8.701808000012079,640000,1397500,23.540282699997306,27187439.002169386,56.61613643993693, -crayon:cpu:standard,english,OK,57.05369000003202,56.57334600005015,136001,740000,7.464656600001263,18219324.382581376,94.5413877633039, -crayon:cpu:standard,code,OK,54.939871000044604,57.17692600001101,312000,1048000,16.698144999992337,18684710.18787675,59.85399477571962, -crayon:cpu:standard,unicode,OK,53.874649000022146,58.702761000006376,216001,660000,7.665993399996296,28176517.866569564,82.10612975852929, -crayon:cpu:standard,mixed,OK,54.17451900001424,57.71157200001653,372501,1397500,22.168933199986895,16802838.307087332,60.11835775564167, -tiktoken:p50k_base,english,OK,0.004907000004550355,0.0004850000436817936,144001,740000,67.69261219999976,2127277.930633626,10.425347334145684, -tiktoken:p50k_base,code,OK,0.005606000001989742,0.0004839999974137754,420000,1048000,217.11489349999624,1934459.6459017552,4.603326227335792, -tiktoken:p50k_base,unicode,OK,0.006116000008660194,0.0004390000185594545,378000,660000,107.76644710000483,3507585.247282251,5.8406402527498456, -tiktoken:p50k_base,mixed,OK,0.005897999983517366,0.000491999969653989,517500,1397500,238.1853491999891,2172677.713965892,5.595473700016287, -tiktoken:cl100k_base,english,OK,0.006203999987519637,0.0004529999841906829,140001,740000,77.06484429999705,1816664.9303151236,9.157469927447217, -tiktoken:cl100k_base,code,OK,0.005318000035003934,0.0004630000489669328,308000,1048000,236.6093924999916,1301723.472368118,4.224053293208914, -tiktoken:cl100k_base,unicode,OK,0.007378999953289167,0.00041700002384459367,306000,660000,192.84278569998605,1586784.7940967705,3.2639284199479937, -tiktoken:cl100k_base,mixed,OK,0.013902999967285723,0.0005999999643790943,410000,1397500,243.1061586000169,1686506.020090482,5.482213469427268, -tiktoken:o200k_base,english,OK,0.0069760000087626395,0.0004720000106317457,140001,740000,152.33581070000355,919028.8176934667,4.632653286825674, -tiktoken:o200k_base,code,OK,0.00585899999805406,0.00041099997361015994,312000,1048000,342.4137217999828,911178.4374758537,2.918839462215151, -tiktoken:o200k_base,unicode,OK,0.006756999937351793,0.0005450000344353612,210000,660000,166.07935479999014,1264455.7793044392,3.7899054315700074, -tiktoken:o200k_base,mixed,OK,0.005947000090600341,0.0006900000926179928,372500,1397500,384.66948759999013,968363.7824358845,3.464688258725745, -crayon:cpu:lite,english,OK,11.818021000067347,7.898499999896558,144001,740000,7.714923399998952,18665253.371150717,91.47453027734804, -crayon:cpu:lite,code,OK,8.684160000029806,8.02162499996939,556000,1048000,16.715822300011496,33261899.416077044,59.79069803781431, -crayon:cpu:lite,unicode,OK,8.600911000030464,8.079272000031779,474000,660000,11.051661499982401,42889478.654476956,56.95297931710334, -crayon:cpu:lite,mixed,OK,8.46338199994534,7.805656000073213,640000,1397500,24.057082800027274,26603391.82933994,55.39989483580376, -crayon:cpu:standard,english,OK,57.284548999973595,56.85542700007318,136001,740000,7.47390640001413,18196775.918914758,94.42438215968168, -crayon:cpu:standard,code,OK,55.02685700002985,56.57900799997151,312000,1048000,16.54470850000962,18857993.17647806,60.40908388281177, -crayon:cpu:standard,unicode,OK,52.882722999925136,55.63082300000133,216001,660000,7.360134700002163,29347424.850789282,85.51814259974617, -crayon:cpu:standard,mixed,OK,52.75064099998872,56.77502999992612,372501,1397500,22.178194400009943,16795821.755437087,60.09325345155857, -tiktoken:p50k_base,english,OK,0.006166000048324349,0.000517999978910666,144001,740000,68.08808929999941,2114922.029395254,10.364793628312775, -tiktoken:p50k_base,code,OK,0.005336999947758159,0.00041600003441999434,420000,1048000,214.1013051999721,1961688.181245402,4.66812046129404, -tiktoken:p50k_base,unicode,OK,0.006585000051018142,0.0004229999603921897,378000,660000,106.76912879999918,3540349.202512205,5.895197009682164, -tiktoken:p50k_base,mixed,OK,0.00516499994773767,0.0004039999339511269,517500,1397500,235.38537899996754,2198522.279500085,5.662033312519034, -tiktoken:cl100k_base,english,OK,0.0059359999795560725,0.0004669999498219113,140001,740000,76.63578599998573,1826835.8335885806,9.20873955857589, -tiktoken:cl100k_base,code,OK,0.006311000106506981,0.000420999981542991,308000,1048000,212.5123685999938,1449327.4063484743,4.703023594240713, -tiktoken:cl100k_base,unicode,OK,0.010396999982731359,0.00044400007936928887,306000,660000,122.41155949998301,2499763.921397002,5.141875909425141, -tiktoken:cl100k_base,mixed,OK,0.006221000035111501,0.0005850000661666854,410000,1397500,240.60089050000215,1704066.8434267342,5.539297275284699, -tiktoken:o200k_base,english,OK,0.0061890000324638095,0.0004290000106266234,140001,740000,147.88289900001246,946701.7548796377,4.77214741469577, -tiktoken:o200k_base,code,OK,0.007050000021990854,0.0004649999709727126,312000,1048000,347.5466003000179,897721.3407660082,2.8757314349528356, -tiktoken:o200k_base,unicode,OK,0.007290999974429724,0.0005510000846697949,210000,660000,167.32717199996614,1255026.2906495694,3.761642782250885, -tiktoken:o200k_base,mixed,OK,0.006225999982234498,0.0004649999709727126,372500,1397500,389.2443974999878,956982.3031300319,3.423966705077049, -crayon:cpu:lite,english,OK,11.785822000092594,7.845040999995945,144001,740000,7.481365799992545,19247956.034998782,94.33023501421735, -crayon:cpu:lite,code,OK,8.199715000046126,7.796341000016582,556000,1048000,16.557010100018488,33580942.249916196,60.36420087662047, -crayon:cpu:lite,unicode,OK,8.307002000037755,7.77954399995906,474000,660000,11.015966299987667,43028454.072207056,57.13752490590223, -crayon:cpu:lite,mixed,OK,8.424993000062386,7.979470000009314,640000,1397500,22.787126700018234,28086033.330366652,58.4874027657321, -crayon:cpu:standard,english,OK,56.081611999957204,54.73750700002711,136001,740000,7.384069699992324,18418163.089676872,95.57317614991616, -crayon:cpu:standard,code,OK,54.299549000006664,54.278538000062326,312000,1048000,16.324630299982346,19112224.550674047,61.223480423616756, -crayon:cpu:standard,unicode,OK,51.12053899995317,55.038874000047144,216001,660000,7.232811699986996,29864043.05263309,87.02356357891311, -crayon:cpu:standard,mixed,OK,52.501413999948454,55.64764499990815,372501,1397500,21.986827800026276,16942007.432266098,60.616286683072204, -tiktoken:p50k_base,english,OK,0.006184999961078574,0.0005029999101680005,144001,740000,67.5318797000159,2132341.060839242,10.450160683746802, -tiktoken:p50k_base,code,OK,0.006159999998089916,0.0004559999524644809,420000,1048000,216.21602799998527,1942501.6909478544,4.622463435475829, -tiktoken:p50k_base,unicode,OK,0.005652000027112081,0.0004470000476430869,378000,660000,107.28266539998685,3523402.3930211407,5.866978103884826, -tiktoken:p50k_base,mixed,OK,0.005496999960996618,0.0004610000132743153,517500,1397500,239.12864910000735,2164107.069343972,5.57340102155787, -tiktoken:cl100k_base,english,OK,0.006750000011379598,0.0005470000132845598,140001,740000,77.6555587999951,1802845.825378425,9.087810390473546, -tiktoken:cl100k_base,code,OK,0.005210000040278828,0.0004349999471742194,308000,1048000,213.01934739999524,1445878.0564267512,4.691830558080906, -tiktoken:cl100k_base,unicode,OK,0.006188999918776972,0.0004959999841958052,306000,660000,121.98645740000984,2508475.174392394,5.1597944742682085, -tiktoken:cl100k_base,mixed,OK,0.006365999979607295,0.0004070000159117626,410000,1397500,242.92543150000938,1687760.7151640859,5.486292023639702, -tiktoken:o200k_base,english,OK,0.006682999924123578,0.00040600002648716327,140001,740000,149.79718980001735,934603.6476846095,4.711163107150247, -tiktoken:o200k_base,code,OK,0.0049780001063481905,0.00044399996568245115,312000,1048000,351.8726216000118,886684.5012871257,2.84037638122885, -tiktoken:o200k_base,unicode,OK,0.006058000053599244,0.0004099999841855606,210000,660000,167.83209560003343,1251250.5385171284,3.7503258633445773, -tiktoken:o200k_base,mixed,OK,0.005523000027096714,0.00045199999476608355,372500,1397500,391.5967607000198,951233.6091190275,3.403398574582915, diff --git a/benchmark_results/20260316_131110/benchmark_results.json b/benchmark_results/20260316_131110/benchmark_results.json deleted file mode 100644 index c6e47e0..0000000 --- a/benchmark_results/20260316_131110/benchmark_results.json +++ /dev/null @@ -1,2602 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 13.420045999993135, - "warm_load_time_ms": 8.633935999995401, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 9.07103429999836, - "tokens_per_sec": 15874815.95125553, - "mb_per_sec": 77.7991760146638, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.61052400000517, - "warm_load_time_ms": 8.313041999997495, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 19.86401780000051, - "tokens_per_sec": 27990309.191123746, - "mb_per_sec": 50.31462887602348, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 12.350933999982772, - "warm_load_time_ms": 8.243020000008983, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 12.465735299994662, - "tokens_per_sec": 38024231.109752744, - "mb_per_sec": 50.49241249558656, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 12.343785000012986, - "warm_load_time_ms": 8.08141199999568, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 26.806486600000312, - "tokens_per_sec": 23874818.418016504, - "mb_per_sec": 49.717811851468774, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 57.44562099999939, - "warm_load_time_ms": 56.62920999998278, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 8.589664400003016, - "tokens_per_sec": 15833098.205786971, - "mb_per_sec": 82.15908809433546, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 59.483790999991015, - "warm_load_time_ms": 62.2261299999991, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 20.4121071000003, - "tokens_per_sec": 15285046.196920818, - "mb_per_sec": 48.96362137908513, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 56.097327999992785, - "warm_load_time_ms": 61.74585100001195, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 8.520422300000519, - "tokens_per_sec": 25350973.507497024, - "mb_per_sec": 73.87251789480982, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 56.56124600000112, - "warm_load_time_ms": 62.25921600000106, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 24.91756019999798, - "tokens_per_sec": 14949336.813482655, - "mb_per_sec": 53.486771838032624, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 871.5962149999825, - "warm_load_time_ms": 0.008374000003641413, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 68.25013059999776, - "tokens_per_sec": 2109900.7244977304, - "mb_per_sec": 10.340185255860128, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.007124999996221959, - "warm_load_time_ms": 0.0005149999822151585, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 219.8429060000052, - "tokens_per_sec": 1910455.09560354, - "mb_per_sec": 4.546203931609813, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.0070859999823369435, - "warm_load_time_ms": 0.0004989999808913126, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 110.32828590000179, - "tokens_per_sec": 3426138.6091197655, - "mb_per_sec": 5.705019738987124, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005673000003980633, - "warm_load_time_ms": 0.0004380000007131457, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 240.371629699996, - "tokens_per_sec": 2152916.30150315, - "mb_per_sec": 5.544580526583486, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 796.1088989999894, - "warm_load_time_ms": 0.005170000008547504, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 84.89410629999554, - "tokens_per_sec": 1649125.0818433713, - "mb_per_sec": 8.312932721698987, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005845000004001122, - "warm_load_time_ms": 0.000515000010636868, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 218.6228982000017, - "tokens_per_sec": 1408818.5754368412, - "mb_per_sec": 4.5715736632465065, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006498000004739879, - "warm_load_time_ms": 0.0004240000066602079, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 128.00079199999175, - "tokens_per_sec": 2390610.2080994914, - "mb_per_sec": 4.917352767849792, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.007242000009455296, - "warm_load_time_ms": 0.0004219999993892998, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 247.372390299995, - "tokens_per_sec": 1657420.2137222437, - "mb_per_sec": 5.38766616420475, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 1123.7693420000028, - "warm_load_time_ms": 0.007920000001604421, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 148.18562679999445, - "tokens_per_sec": 944767.7418064222, - "mb_per_sec": 4.76239841461231, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005945999987488904, - "warm_load_time_ms": 0.0004309999894758221, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 527.5963346000026, - "tokens_per_sec": 591361.1970722711, - "mb_per_sec": 1.8943472841817297, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006991999981664776, - "warm_load_time_ms": 0.0004199999921183917, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 250.5198507000017, - "tokens_per_sec": 838256.9262005335, - "mb_per_sec": 2.51247574621088, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005476999945130956, - "warm_load_time_ms": 0.00038700000004610047, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 399.6432679999941, - "tokens_per_sec": 932081.2580283611, - "mb_per_sec": 3.334873783430662, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 12.255995000032271, - "warm_load_time_ms": 8.353256999953373, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.629351100001713, - "tokens_per_sec": 18874606.51797342, - "mb_per_sec": 92.5005265704008, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.631075999971927, - "warm_load_time_ms": 8.182028000021546, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 17.14357649999556, - "tokens_per_sec": 32431972.406699616, - "mb_per_sec": 58.29884351109635, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.595698000021912, - "warm_load_time_ms": 8.26907499998697, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.279014600006576, - "tokens_per_sec": 42024947.81766872, - "mb_per_sec": 55.804968000285946, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.619954999971924, - "warm_load_time_ms": 8.18737700001293, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 27.430875499999274, - "tokens_per_sec": 23331373.43720644, - "mb_per_sec": 48.586121765518186, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 57.70983200000046, - "warm_load_time_ms": 57.04072999998289, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.748942700010275, - "tokens_per_sec": 17550910.526131477, - "mb_per_sec": 91.07294007215839, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 55.32362400003876, - "warm_load_time_ms": 56.88838099996474, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 18.391508200005546, - "tokens_per_sec": 16964350.971493784, - "mb_per_sec": 54.34305184353769, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 53.574158000003536, - "warm_load_time_ms": 56.808552000006785, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.41275420001557, - "tokens_per_sec": 29139101.900822006, - "mb_per_sec": 84.91109132241333, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 53.982016999952975, - "warm_load_time_ms": 56.91404300000613, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 22.914766900004224, - "tokens_per_sec": 16255936.690323973, - "mb_per_sec": 58.16161530220448, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.007771000014145102, - "warm_load_time_ms": 0.0005619999683403876, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 68.49571729999866, - "tokens_per_sec": 2102335.8200528375, - "mb_per_sec": 10.3031112302935, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.006550999955834413, - "warm_load_time_ms": 0.000489000001380191, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 219.48089210000603, - "tokens_per_sec": 1913606.2186617495, - "mb_per_sec": 4.553702484216039, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.0067070000113744754, - "warm_load_time_ms": 0.0005200000146032835, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 107.88554559999852, - "tokens_per_sec": 3503713.1053819796, - "mb_per_sec": 5.834192572578821, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.007401000004847447, - "warm_load_time_ms": 0.00043800002913485514, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 241.58617579999486, - "tokens_per_sec": 2142092.767876046, - "mb_per_sec": 5.516705799760264, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.010244000009151932, - "warm_load_time_ms": 0.00043000000005122274, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 78.11836440000661, - "tokens_per_sec": 1792165.0187543887, - "mb_per_sec": 9.03397043142093, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.008102000037979451, - "warm_load_time_ms": 0.0004949999947712058, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 216.04720779998843, - "tokens_per_sec": 1425614.351309457, - "mb_per_sec": 4.626075447913303, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006573999996817292, - "warm_load_time_ms": 0.00040900005160438013, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 123.78520560000084, - "tokens_per_sec": 2472024.0073664985, - "mb_per_sec": 5.084816442944299, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006136000024525856, - "warm_load_time_ms": 0.0004350000040176383, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 243.61621470000046, - "tokens_per_sec": 1682975.0043727253, - "mb_per_sec": 5.470735430393878, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006606999988889584, - "warm_load_time_ms": 0.0008569999749852286, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 147.9927857000007, - "tokens_per_sec": 945998.8156706436, - "mb_per_sec": 4.76860402892343, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.006286000029831484, - "warm_load_time_ms": 0.0004669999498219113, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 348.46320080000623, - "tokens_per_sec": 895359.9670889392, - "mb_per_sec": 2.8681670870817877, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.005886999986159935, - "warm_load_time_ms": 0.0004969999736204045, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 167.48856810000348, - "tokens_per_sec": 1253816.9164752422, - "mb_per_sec": 3.758017970828374, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.012467000033211662, - "warm_load_time_ms": 0.00045899997758169775, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 386.2139582999987, - "tokens_per_sec": 964491.2929600899, - "mb_per_sec": 3.450832960683645, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 14.46436700001641, - "warm_load_time_ms": 8.621505999997225, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.970677099996237, - "tokens_per_sec": 18066344.70239272, - "mb_per_sec": 88.53940327616058, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.487282999965373, - "warm_load_time_ms": 8.221886000001177, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 17.167496500002244, - "tokens_per_sec": 32386783.943711728, - "mb_per_sec": 58.217613942348514, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.32785199997943, - "warm_load_time_ms": 7.899138000027506, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.294528399997716, - "tokens_per_sec": 41967223.70454138, - "mb_per_sec": 55.72831609580549, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.416824999983419, - "warm_load_time_ms": 8.190843000022596, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 25.39328400000045, - "tokens_per_sec": 25203514.441062003, - "mb_per_sec": 52.48473797944806, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 59.122182999999495, - "warm_load_time_ms": 56.704432000003635, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.6757321999991746, - "tokens_per_sec": 17718309.661717307, - "mb_per_sec": 91.94158625553675, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 57.78209900000775, - "warm_load_time_ms": 57.2303270000134, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 17.449783000000707, - "tokens_per_sec": 17879878.506224826, - "mb_per_sec": 57.27582306288333, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 52.829341000006025, - "warm_load_time_ms": 55.68740900002922, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.266771999991306, - "tokens_per_sec": 29724477.38834498, - "mb_per_sec": 86.61687043832916, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 54.2102670000304, - "warm_load_time_ms": 56.79054400002315, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 22.231868899996243, - "tokens_per_sec": 16755271.528254783, - "mb_per_sec": 59.94817004241868, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.005590999990090495, - "warm_load_time_ms": 0.0004799999828719592, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 67.79733839999267, - "tokens_per_sec": 2123991.9353532554, - "mb_per_sec": 10.409243353728785, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.00664499998492829, - "warm_load_time_ms": 0.00041600003441999434, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 215.4200641999921, - "tokens_per_sec": 1949679.1144304904, - "mb_per_sec": 4.639543151680978, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006171000052290765, - "warm_load_time_ms": 0.0005260000079942984, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 107.30631869999456, - "tokens_per_sec": 3522625.7370435647, - "mb_per_sec": 5.865684858576338, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005192000003262365, - "warm_load_time_ms": 0.0004579999881570984, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 236.72034209999424, - "tokens_per_sec": 2186123.91064136, - "mb_per_sec": 5.630102784384945, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.007005999975717714, - "warm_load_time_ms": 0.0004910000370728085, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 77.01460709998855, - "tokens_per_sec": 1817849.9543370495, - "mb_per_sec": 9.16344341307079, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.006001999963700655, - "warm_load_time_ms": 0.0004839999974137754, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 212.1594686999913, - "tokens_per_sec": 1451738.1754737238, - "mb_per_sec": 4.710846467131028, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.005649999991419463, - "warm_load_time_ms": 0.00044099999740865314, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 121.5911033999987, - "tokens_per_sec": 2516631.492300474, - "mb_per_sec": 5.176571568377853, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.0057810000271274475, - "warm_load_time_ms": 0.0004139999987273768, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 240.63990579999768, - "tokens_per_sec": 1703790.5605762748, - "mb_per_sec": 5.53839918091319, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.007540000012795645, - "warm_load_time_ms": 0.0004569999987324991, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 147.87910310000711, - "tokens_per_sec": 946726.0557113377, - "mb_per_sec": 4.772269910667257, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.0067850000391445064, - "warm_load_time_ms": 0.0004030000013699464, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 344.18638970000757, - "tokens_per_sec": 906485.5826284671, - "mb_per_sec": 2.9038065231599366, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.005466999994041544, - "warm_load_time_ms": 0.0005240000291450997, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 165.9669846999975, - "tokens_per_sec": 1265311.8954929302, - "mb_per_sec": 3.7924714362068808, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005138000005899812, - "warm_load_time_ms": 0.00040200001194534707, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 386.74107549999803, - "tokens_per_sec": 963176.7184760851, - "mb_per_sec": 3.44612957249156, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 11.859816000026058, - "warm_load_time_ms": 8.044323999968128, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.576637799991204, - "tokens_per_sec": 19005923.70934865, - "mb_per_sec": 93.14408485270924, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.248379999997724, - "warm_load_time_ms": 7.75930600002539, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.88808420000214, - "tokens_per_sec": 32922621.264520314, - "mb_per_sec": 59.18082073475352, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.604470999955538, - "warm_load_time_ms": 8.002074999978959, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.156769700005498, - "tokens_per_sec": 42485415.82782393, - "mb_per_sec": 56.4164239069858, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.456352999985484, - "warm_load_time_ms": 8.093239999993784, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 23.72167619999459, - "tokens_per_sec": 26979543.71370038, - "mb_per_sec": 56.18320754154963, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 57.44187900000952, - "warm_load_time_ms": 57.31347900001538, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.547879199995577, - "tokens_per_sec": 18018438.874866955, - "mb_per_sec": 93.49897838071368, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 55.624494000028335, - "warm_load_time_ms": 56.39792900001339, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.869116499987058, - "tokens_per_sec": 18495337.322511192, - "mb_per_sec": 59.24736387910515, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 52.97563800002081, - "warm_load_time_ms": 56.39938499996333, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.500860099992224, - "tokens_per_sec": 28796830.912794113, - "mb_per_sec": 83.91371661881516, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 53.65096499997435, - "warm_load_time_ms": 56.49196399997436, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 21.90999329999954, - "tokens_per_sec": 17001420.078024752, - "mb_per_sec": 60.828857358790806, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006266999946547003, - "warm_load_time_ms": 0.0005079999709778349, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 67.73963460000232, - "tokens_per_sec": 2125801.2513695355, - "mb_per_sec": 10.41811043575663, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.006395000013981189, - "warm_load_time_ms": 0.00042300001723560854, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 215.189668499994, - "tokens_per_sec": 1951766.5644808207, - "mb_per_sec": 4.6445105406804315, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006347999999434251, - "warm_load_time_ms": 0.00045899997758169775, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 107.80626470000243, - "tokens_per_sec": 3506289.7416201034, - "mb_per_sec": 5.838483047155522, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005572000020492851, - "warm_load_time_ms": 0.0004800000397153781, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 240.88481109999975, - "tokens_per_sec": 2148329.7250533886, - "mb_per_sec": 5.53276834305862, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.0059879999980694265, - "warm_load_time_ms": 0.0004139999987273768, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 77.52655099998833, - "tokens_per_sec": 1805845.8449934279, - "mb_per_sec": 9.102932931206126, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.03849000000855085, - "warm_load_time_ms": 0.0005480000027091592, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 212.68630399998187, - "tokens_per_sec": 1448142.142711861, - "mb_per_sec": 4.699177449591842, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006583000015325524, - "warm_load_time_ms": 0.00044500001195046934, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 122.85836879999579, - "tokens_per_sec": 2490672.820979294, - "mb_per_sec": 5.123176019476392, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.0062469999875247595, - "warm_load_time_ms": 0.0005079999709778349, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 241.95004649999987, - "tokens_per_sec": 1694564.667091313, - "mb_per_sec": 5.508409179734277, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006681000002117798, - "warm_load_time_ms": 0.00043800002913485514, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 149.0840187000117, - "tokens_per_sec": 939074.4978622515, - "mb_per_sec": 4.733699831104497, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.00601800002186792, - "warm_load_time_ms": 0.0004799999828719592, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 344.87705079999387, - "tokens_per_sec": 904670.2274804002, - "mb_per_sec": 2.8979912733403825, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006578000011359109, - "warm_load_time_ms": 0.0004529999841906829, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 166.56413929999871, - "tokens_per_sec": 1260775.5840035228, - "mb_per_sec": 3.778874921536786, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005728999951770675, - "warm_load_time_ms": 0.0005079999709778349, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 386.02468130000034, - "tokens_per_sec": 964964.2057744759, - "mb_per_sec": 3.452524985421789, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 11.945968000020457, - "warm_load_time_ms": 8.068296999965696, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.684993499987058, - "tokens_per_sec": 18737946.88313562, - "mb_per_sec": 91.8307860822281, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.50575999999137, - "warm_load_time_ms": 8.087123999985124, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.832955599988964, - "tokens_per_sec": 33030444.16040428, - "mb_per_sec": 59.37464027971447, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.485736000011457, - "warm_load_time_ms": 7.785606999959782, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.008489499999996, - "tokens_per_sec": 43057678.34905962, - "mb_per_sec": 57.176331850807074, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.317911999995431, - "warm_load_time_ms": 7.874027000013939, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 23.156396000007362, - "tokens_per_sec": 27638152.32732229, - "mb_per_sec": 57.55471866940394, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 56.3960169999973, - "warm_load_time_ms": 55.77992599995696, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.532129799994891, - "tokens_per_sec": 18056114.752575327, - "mb_per_sec": 93.69448122642598, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 55.15872899997021, - "warm_load_time_ms": 55.88644499999873, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.60530089999952, - "tokens_per_sec": 18789180.74890224, - "mb_per_sec": 60.18865238351561, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 52.26000599998315, - "warm_load_time_ms": 55.77487499999734, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.332337400015376, - "tokens_per_sec": 29458682.575019944, - "mb_per_sec": 85.8423466474422, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 52.8641419999758, - "warm_load_time_ms": 56.416287999979886, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 22.371619800003373, - "tokens_per_sec": 16650604.798850723, - "mb_per_sec": 59.57368617437051, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006202999998095038, - "warm_load_time_ms": 0.0004969999736204045, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 84.53547129999492, - "tokens_per_sec": 1703438.778840861, - "mb_per_sec": 8.348199676277991, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.013772000045264576, - "warm_load_time_ms": 0.0006299999881775875, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 331.9959505999975, - "tokens_per_sec": 1265075.671076583, - "mb_per_sec": 3.010430343465031, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.00607499998750427, - "warm_load_time_ms": 0.0007910000476840651, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 107.05100760001187, - "tokens_per_sec": 3531027.0166944047, - "mb_per_sec": 5.879674212688636, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005955999995421735, - "warm_load_time_ms": 0.00043699998286683694, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 236.48260150000056, - "tokens_per_sec": 2188321.663908957, - "mb_per_sec": 5.635762837198538, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006967000047097827, - "warm_load_time_ms": 0.0005080000278212538, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 76.65469670000107, - "tokens_per_sec": 1826385.1535139927, - "mb_per_sec": 9.20646776416787, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005427000019153638, - "warm_load_time_ms": 0.0005530000066755747, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 209.65246490000027, - "tokens_per_sec": 1469097.9194874212, - "mb_per_sec": 4.767178311356685, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006586000040442741, - "warm_load_time_ms": 0.0005190000251786842, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 121.59806469999808, - "tokens_per_sec": 2516487.4190633795, - "mb_per_sec": 5.176275217710228, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006361999965065479, - "warm_load_time_ms": 0.0004229999603921897, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 240.8612956000013, - "tokens_per_sec": 1702224.5063436327, - "mb_per_sec": 5.533308512095071, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.007159999995565158, - "warm_load_time_ms": 0.0005079999709778349, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 146.5338551000059, - "tokens_per_sec": 955417.4351343765, - "mb_per_sec": 4.816081537327934, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005568999995375634, - "warm_load_time_ms": 0.00043200003574384027, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 341.9993831000113, - "tokens_per_sec": 912282.3473303209, - "mb_per_sec": 2.9223756912493593, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.0060189999544491, - "warm_load_time_ms": 0.0004590000344251166, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 166.09841920000008, - "tokens_per_sec": 1264310.6479366175, - "mb_per_sec": 3.789470434816304, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005809000015233323, - "warm_load_time_ms": 0.0004269999749340059, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 385.07791709998287, - "tokens_per_sec": 967336.6959219397, - "mb_per_sec": 3.4610134676502167, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 12.146585999971649, - "warm_load_time_ms": 8.18777499995349, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.5930362000008245, - "tokens_per_sec": 18964877.317453638, - "mb_per_sec": 92.94292501075503, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.266219999995883, - "warm_load_time_ms": 7.896754000000783, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.99046889999636, - "tokens_per_sec": 32724229.288346432, - "mb_per_sec": 58.82419664085694, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.481736000021556, - "warm_load_time_ms": 7.952439000007416, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.347498700001779, - "tokens_per_sec": 41771320.05310789, - "mb_per_sec": 55.46817545156681, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.46451799998249, - "warm_load_time_ms": 8.242848000008962, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 24.098211299997274, - "tokens_per_sec": 26557987.72915865, - "mb_per_sec": 55.30534364506486, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 57.78302799996027, - "warm_load_time_ms": 57.101206000027105, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.574477499991872, - "tokens_per_sec": 17955165.88439875, - "mb_per_sec": 93.17065027143883, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 55.734033000021554, - "warm_load_time_ms": 57.81873300003326, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.75580560000185, - "tokens_per_sec": 18620411.781333007, - "mb_per_sec": 59.6480233450333, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 53.034622000041054, - "warm_load_time_ms": 56.92579700001943, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.35246310000548, - "tokens_per_sec": 29378046.113531534, - "mb_per_sec": 85.60737269496204, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 53.62937799998235, - "warm_load_time_ms": 56.189039999992474, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 22.484254099992995, - "tokens_per_sec": 16567194.016906083, - "mb_per_sec": 59.27525330618593, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.00639499995713777, - "warm_load_time_ms": 0.00045899997758169775, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 68.15755910000121, - "tokens_per_sec": 2112766.388666014, - "mb_per_sec": 10.354229280793191, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.0067010000179834606, - "warm_load_time_ms": 0.0005269999974188977, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 214.87836619999712, - "tokens_per_sec": 1954594.161466617, - "mb_per_sec": 4.651239216252769, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.01659599996628458, - "warm_load_time_ms": 0.0004880000119555916, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 107.26061450000088, - "tokens_per_sec": 3524126.7427196857, - "mb_per_sec": 5.868184251621268, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006431000031170697, - "warm_load_time_ms": 0.0007040000014058023, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 236.91750270000966, - "tokens_per_sec": 2184304.638122369, - "mb_per_sec": 5.625417463838901, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006840999958512839, - "warm_load_time_ms": 0.0005940000278314983, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 77.5616793999859, - "tokens_per_sec": 1805027.9607538444, - "mb_per_sec": 9.09881012891984, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.00674799997568698, - "warm_load_time_ms": 0.0004689999855145288, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 209.97482439998976, - "tokens_per_sec": 1466842.5173356878, - "mb_per_sec": 4.7598595995958775, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006465000012667588, - "warm_load_time_ms": 0.000392000004012516, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 121.70541190000108, - "tokens_per_sec": 2514267.8145769234, - "mb_per_sec": 5.171709614238768, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006374999998115527, - "warm_load_time_ms": 0.0004489999696488667, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 239.08225989999323, - "tokens_per_sec": 1714890.934072234, - "mb_per_sec": 5.574482430169517, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006678999966425181, - "warm_load_time_ms": 0.0004799999828719592, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 146.80623299998956, - "tokens_per_sec": 953644.7951771227, - "mb_per_sec": 4.807145989098944, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.0051750000125139195, - "warm_load_time_ms": 0.00044500001195046934, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 342.70248529999776, - "tokens_per_sec": 910410.6721807951, - "mb_per_sec": 2.9163800277632728, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.0055859999861240794, - "warm_load_time_ms": 0.00046999997493912815, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 165.92932179999593, - "tokens_per_sec": 1265599.0979889918, - "mb_per_sec": 3.793332257374058, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.007171999982347188, - "warm_load_time_ms": 0.00043299996832502075, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 384.91586170000005, - "tokens_per_sec": 967743.9593027817, - "mb_per_sec": 3.4624706066711157, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 11.53909399999975, - "warm_load_time_ms": 7.790105999958996, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.39537980001046, - "tokens_per_sec": 19471751.809122276, - "mb_per_sec": 95.42701162415308, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.380036000005475, - "warm_load_time_ms": 7.807446000015261, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.601938900004143, - "tokens_per_sec": 33490064.22375529, - "mb_per_sec": 60.20084097487556, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.321920000014416, - "warm_load_time_ms": 7.639664000009816, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 10.848814999997103, - "tokens_per_sec": 43691407.77127517, - "mb_per_sec": 58.017861750642176, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.218655000007402, - "warm_load_time_ms": 7.681361000038578, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 23.521902400000272, - "tokens_per_sec": 27208683.596952286, - "mb_per_sec": 56.660376976044205, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 55.9084590000225, - "warm_load_time_ms": 55.01668699997708, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.247118899988436, - "tokens_per_sec": 18766216.185609568, - "mb_per_sec": 97.37924875797898, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 54.783281999959854, - "warm_load_time_ms": 54.293733000008615, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.26572840000904, - "tokens_per_sec": 19181434.260258924, - "mb_per_sec": 61.4451845632191, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 51.37580899997829, - "warm_load_time_ms": 54.80919600000789, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.13365659998999, - "tokens_per_sec": 30279141.835941903, - "mb_per_sec": 88.23315784909077, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 52.157340000007935, - "warm_load_time_ms": 55.81748999998126, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 22.280753100005768, - "tokens_per_sec": 16718510.291283807, - "mb_per_sec": 59.81664314467851, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.004771000021719374, - "warm_load_time_ms": 0.0004709999643637275, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 67.3320732000036, - "tokens_per_sec": 2138668.7377389753, - "mb_per_sec": 10.481171314068304, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.004755999952976708, - "warm_load_time_ms": 0.00044099999740865314, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 214.3942374999881, - "tokens_per_sec": 1959007.876785976, - "mb_per_sec": 4.661742289569726, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.005805999990116106, - "warm_load_time_ms": 0.0005059999921286362, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 106.82626730001061, - "tokens_per_sec": 3538455.564851159, - "mb_per_sec": 5.892043827202623, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005986000019220228, - "warm_load_time_ms": 0.0004770000145981612, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 234.7759601000064, - "tokens_per_sec": 2204229.0862299656, - "mb_per_sec": 5.67673051623354, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006902999984959024, - "warm_load_time_ms": 0.000596000006680697, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 76.99923400001012, - "tokens_per_sec": 1818212.8928708772, - "mb_per_sec": 9.165272918695948, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005643000008603849, - "warm_load_time_ms": 0.00046999997493912815, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 209.2496414999914, - "tokens_per_sec": 1471926.0582341913, - "mb_per_sec": 4.776355536044209, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.00608600004170512, - "warm_load_time_ms": 0.00044500001195046934, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 121.43914309998536, - "tokens_per_sec": 2519780.625823906, - "mb_per_sec": 5.183049161586195, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005396000005930546, - "warm_load_time_ms": 0.000392000004012516, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 239.10487489999355, - "tokens_per_sec": 1714728.736381824, - "mb_per_sec": 5.573955184875113, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.0062280000179271156, - "warm_load_time_ms": 0.00048499998683837475, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 146.86210019999066, - "tokens_per_sec": 953282.0231315806, - "mb_per_sec": 4.80531732271026, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.012413000035849109, - "warm_load_time_ms": 0.0005310000119607139, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 341.3278896999884, - "tokens_per_sec": 914077.077833381, - "mb_per_sec": 2.9281248727498403, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.00512899998739158, - "warm_load_time_ms": 0.0004539999736152822, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 165.44453200000362, - "tokens_per_sec": 1269307.588841893, - "mb_per_sec": 3.804447576593896, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006564999978309061, - "warm_load_time_ms": 0.00038000001723048626, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 383.76850569999306, - "tokens_per_sec": 970637.2317357326, - "mb_per_sec": 3.4728223848040445, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 11.782717000016873, - "warm_load_time_ms": 7.8747029999703955, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.522683499996674, - "tokens_per_sec": 19142238.27176348, - "mb_per_sec": 93.8121342126034, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.217624999986128, - "warm_load_time_ms": 7.702755999957844, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.541529200003424, - "tokens_per_sec": 33612370.00989515, - "mb_per_sec": 60.420694574812536, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.531998000023577, - "warm_load_time_ms": 7.964495000010174, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.138282199999594, - "tokens_per_sec": 42555933.804587685, - "mb_per_sec": 56.510064795103496, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.412228999986837, - "warm_load_time_ms": 8.701808000012079, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 23.540282699997306, - "tokens_per_sec": 27187439.002169386, - "mb_per_sec": 56.61613643993693, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 57.05369000003202, - "warm_load_time_ms": 56.57334600005015, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.464656600001263, - "tokens_per_sec": 18219324.382581376, - "mb_per_sec": 94.5413877633039, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 54.939871000044604, - "warm_load_time_ms": 57.17692600001101, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.698144999992337, - "tokens_per_sec": 18684710.18787675, - "mb_per_sec": 59.85399477571962, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 53.874649000022146, - "warm_load_time_ms": 58.702761000006376, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.665993399996296, - "tokens_per_sec": 28176517.866569564, - "mb_per_sec": 82.10612975852929, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 54.17451900001424, - "warm_load_time_ms": 57.71157200001653, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 22.168933199986895, - "tokens_per_sec": 16802838.307087332, - "mb_per_sec": 60.11835775564167, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.004907000004550355, - "warm_load_time_ms": 0.0004850000436817936, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 67.69261219999976, - "tokens_per_sec": 2127277.930633626, - "mb_per_sec": 10.425347334145684, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005606000001989742, - "warm_load_time_ms": 0.0004839999974137754, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 217.11489349999624, - "tokens_per_sec": 1934459.6459017552, - "mb_per_sec": 4.603326227335792, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006116000008660194, - "warm_load_time_ms": 0.0004390000185594545, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 107.76644710000483, - "tokens_per_sec": 3507585.247282251, - "mb_per_sec": 5.8406402527498456, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005897999983517366, - "warm_load_time_ms": 0.000491999969653989, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 238.1853491999891, - "tokens_per_sec": 2172677.713965892, - "mb_per_sec": 5.595473700016287, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006203999987519637, - "warm_load_time_ms": 0.0004529999841906829, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 77.06484429999705, - "tokens_per_sec": 1816664.9303151236, - "mb_per_sec": 9.157469927447217, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005318000035003934, - "warm_load_time_ms": 0.0004630000489669328, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 236.6093924999916, - "tokens_per_sec": 1301723.472368118, - "mb_per_sec": 4.224053293208914, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.007378999953289167, - "warm_load_time_ms": 0.00041700002384459367, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 192.84278569998605, - "tokens_per_sec": 1586784.7940967705, - "mb_per_sec": 3.2639284199479937, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.013902999967285723, - "warm_load_time_ms": 0.0005999999643790943, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 243.1061586000169, - "tokens_per_sec": 1686506.020090482, - "mb_per_sec": 5.482213469427268, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.0069760000087626395, - "warm_load_time_ms": 0.0004720000106317457, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 152.33581070000355, - "tokens_per_sec": 919028.8176934667, - "mb_per_sec": 4.632653286825674, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.00585899999805406, - "warm_load_time_ms": 0.00041099997361015994, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 342.4137217999828, - "tokens_per_sec": 911178.4374758537, - "mb_per_sec": 2.918839462215151, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006756999937351793, - "warm_load_time_ms": 0.0005450000344353612, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 166.07935479999014, - "tokens_per_sec": 1264455.7793044392, - "mb_per_sec": 3.7899054315700074, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005947000090600341, - "warm_load_time_ms": 0.0006900000926179928, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 384.66948759999013, - "tokens_per_sec": 968363.7824358845, - "mb_per_sec": 3.464688258725745, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 11.818021000067347, - "warm_load_time_ms": 7.898499999896558, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.714923399998952, - "tokens_per_sec": 18665253.371150717, - "mb_per_sec": 91.47453027734804, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.684160000029806, - "warm_load_time_ms": 8.02162499996939, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.715822300011496, - "tokens_per_sec": 33261899.416077044, - "mb_per_sec": 59.79069803781431, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.600911000030464, - "warm_load_time_ms": 8.079272000031779, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.051661499982401, - "tokens_per_sec": 42889478.654476956, - "mb_per_sec": 56.95297931710334, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.46338199994534, - "warm_load_time_ms": 7.805656000073213, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 24.057082800027274, - "tokens_per_sec": 26603391.82933994, - "mb_per_sec": 55.39989483580376, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 57.284548999973595, - "warm_load_time_ms": 56.85542700007318, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.47390640001413, - "tokens_per_sec": 18196775.918914758, - "mb_per_sec": 94.42438215968168, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 55.02685700002985, - "warm_load_time_ms": 56.57900799997151, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.54470850000962, - "tokens_per_sec": 18857993.17647806, - "mb_per_sec": 60.40908388281177, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 52.882722999925136, - "warm_load_time_ms": 55.63082300000133, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.360134700002163, - "tokens_per_sec": 29347424.850789282, - "mb_per_sec": 85.51814259974617, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 52.75064099998872, - "warm_load_time_ms": 56.77502999992612, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 22.178194400009943, - "tokens_per_sec": 16795821.755437087, - "mb_per_sec": 60.09325345155857, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006166000048324349, - "warm_load_time_ms": 0.000517999978910666, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 68.08808929999941, - "tokens_per_sec": 2114922.029395254, - "mb_per_sec": 10.364793628312775, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005336999947758159, - "warm_load_time_ms": 0.00041600003441999434, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 214.1013051999721, - "tokens_per_sec": 1961688.181245402, - "mb_per_sec": 4.66812046129404, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006585000051018142, - "warm_load_time_ms": 0.0004229999603921897, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 106.76912879999918, - "tokens_per_sec": 3540349.202512205, - "mb_per_sec": 5.895197009682164, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.00516499994773767, - "warm_load_time_ms": 0.0004039999339511269, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 235.38537899996754, - "tokens_per_sec": 2198522.279500085, - "mb_per_sec": 5.662033312519034, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.0059359999795560725, - "warm_load_time_ms": 0.0004669999498219113, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 76.63578599998573, - "tokens_per_sec": 1826835.8335885806, - "mb_per_sec": 9.20873955857589, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.006311000106506981, - "warm_load_time_ms": 0.000420999981542991, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 212.5123685999938, - "tokens_per_sec": 1449327.4063484743, - "mb_per_sec": 4.703023594240713, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.010396999982731359, - "warm_load_time_ms": 0.00044400007936928887, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 122.41155949998301, - "tokens_per_sec": 2499763.921397002, - "mb_per_sec": 5.141875909425141, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006221000035111501, - "warm_load_time_ms": 0.0005850000661666854, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 240.60089050000215, - "tokens_per_sec": 1704066.8434267342, - "mb_per_sec": 5.539297275284699, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.0061890000324638095, - "warm_load_time_ms": 0.0004290000106266234, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 147.88289900001246, - "tokens_per_sec": 946701.7548796377, - "mb_per_sec": 4.77214741469577, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.007050000021990854, - "warm_load_time_ms": 0.0004649999709727126, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 347.5466003000179, - "tokens_per_sec": 897721.3407660082, - "mb_per_sec": 2.8757314349528356, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.007290999974429724, - "warm_load_time_ms": 0.0005510000846697949, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 167.32717199996614, - "tokens_per_sec": 1255026.2906495694, - "mb_per_sec": 3.761642782250885, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006225999982234498, - "warm_load_time_ms": 0.0004649999709727126, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 389.2443974999878, - "tokens_per_sec": 956982.3031300319, - "mb_per_sec": 3.423966705077049, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 11.785822000092594, - "warm_load_time_ms": 7.845040999995945, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.481365799992545, - "tokens_per_sec": 19247956.034998782, - "mb_per_sec": 94.33023501421735, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.199715000046126, - "warm_load_time_ms": 7.796341000016582, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.557010100018488, - "tokens_per_sec": 33580942.249916196, - "mb_per_sec": 60.36420087662047, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 8.307002000037755, - "warm_load_time_ms": 7.77954399995906, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 11.015966299987667, - "tokens_per_sec": 43028454.072207056, - "mb_per_sec": 57.13752490590223, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 8.424993000062386, - "warm_load_time_ms": 7.979470000009314, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 22.787126700018234, - "tokens_per_sec": 28086033.330366652, - "mb_per_sec": 58.4874027657321, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 56.081611999957204, - "warm_load_time_ms": 54.73750700002711, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.384069699992324, - "tokens_per_sec": 18418163.089676872, - "mb_per_sec": 95.57317614991616, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 54.299549000006664, - "warm_load_time_ms": 54.278538000062326, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.324630299982346, - "tokens_per_sec": 19112224.550674047, - "mb_per_sec": 61.223480423616756, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 51.12053899995317, - "warm_load_time_ms": 55.038874000047144, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.232811699986996, - "tokens_per_sec": 29864043.05263309, - "mb_per_sec": 87.02356357891311, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 52.501413999948454, - "warm_load_time_ms": 55.64764499990815, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 21.986827800026276, - "tokens_per_sec": 16942007.432266098, - "mb_per_sec": 60.616286683072204, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006184999961078574, - "warm_load_time_ms": 0.0005029999101680005, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 67.5318797000159, - "tokens_per_sec": 2132341.060839242, - "mb_per_sec": 10.450160683746802, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.006159999998089916, - "warm_load_time_ms": 0.0004559999524644809, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 216.21602799998527, - "tokens_per_sec": 1942501.6909478544, - "mb_per_sec": 4.622463435475829, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.005652000027112081, - "warm_load_time_ms": 0.0004470000476430869, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 107.28266539998685, - "tokens_per_sec": 3523402.3930211407, - "mb_per_sec": 5.866978103884826, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005496999960996618, - "warm_load_time_ms": 0.0004610000132743153, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 239.12864910000735, - "tokens_per_sec": 2164107.069343972, - "mb_per_sec": 5.57340102155787, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006750000011379598, - "warm_load_time_ms": 0.0005470000132845598, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 77.6555587999951, - "tokens_per_sec": 1802845.825378425, - "mb_per_sec": 9.087810390473546, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.005210000040278828, - "warm_load_time_ms": 0.0004349999471742194, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 213.01934739999524, - "tokens_per_sec": 1445878.0564267512, - "mb_per_sec": 4.691830558080906, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006188999918776972, - "warm_load_time_ms": 0.0004959999841958052, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 121.98645740000984, - "tokens_per_sec": 2508475.174392394, - "mb_per_sec": 5.1597944742682085, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.006365999979607295, - "warm_load_time_ms": 0.0004070000159117626, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 242.92543150000938, - "tokens_per_sec": 1687760.7151640859, - "mb_per_sec": 5.486292023639702, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 0.006682999924123578, - "warm_load_time_ms": 0.00040600002648716327, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 149.79718980001735, - "tokens_per_sec": 934603.6476846095, - "mb_per_sec": 4.711163107150247, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.0049780001063481905, - "warm_load_time_ms": 0.00044399996568245115, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 351.8726216000118, - "tokens_per_sec": 886684.5012871257, - "mb_per_sec": 2.84037638122885, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006058000053599244, - "warm_load_time_ms": 0.0004099999841855606, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 167.83209560003343, - "tokens_per_sec": 1251250.5385171284, - "mb_per_sec": 3.7503258633445773, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005523000027096714, - "warm_load_time_ms": 0.00045199999476608355, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 391.5967607000198, - "tokens_per_sec": 951233.6091190275, - "mb_per_sec": 3.403398574582915, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_131110/benchmark_summary.csv b/benchmark_results/20260316_131110/benchmark_summary.csv deleted file mode 100644 index 3be8118..0000000 --- a/benchmark_results/20260316_131110/benchmark_summary.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,n,tokens_per_sec_mean,tokens_per_sec_std,cold_load_time_ms_mean,cold_load_time_ms_std,warm_load_time_ms_mean,warm_load_time_ms_std,mb_per_sec_mean,mb_per_sec_std,tokens_produced_mean,tokens_produced_std -crayon:cpu:lite,code,10,32543163.61544498,1660418.469066782,8.423077899999498,0.184571569644575,7.978830799999059,0.21588091628542938,58.49871784489161,2.984723694173529,556000.0,0.0 -crayon:cpu:lite,english,10,18605171.456859484,1031803.966051145,12.301843200023654,0.9205535110367989,8.13174449996552,0.31186301379951287,91.18008129352394,5.05665694732621,144001.0,0.0 -crayon:cpu:lite,mixed,10,26267093.78252945,1602078.7282628026,8.81386069999337,1.2445905519076692,8.083804200018108,0.28357285227868667,54.69957524699704,3.336228464167218,640000.0,0.0 -crayon:cpu:lite,unicode,10,42149609.116450116,1563828.113027747,8.860825800007888,1.2318979023066992,7.961432899998044,0.20027127594089345,55.97050585697889,2.0766088320702214,474000.0,0.0 -crayon:cpu:standard,code,10,18187056.770267364,1211012.2796866265,55.81563290000986,1.5913846515152064,56.87761500000761,2.2221049765449994,58.25982795385274,3.879317470427001,312000.0,0.0 -crayon:cpu:standard,english,10,17873251.748225935,793733.3382264881,57.222686999995176,0.9429351112404759,56.375195000009626,0.8928807337267614,92.74559191314898,4.11873951712912,136001.0,0.0 -crayon:cpu:standard,mixed,10,16543894.171191728,597306.6951312533,53.648192899987635,1.2538254564721554,57.10128319998091,1.90463315356463,59.19188950569539,2.1370852311656443,372501.0,0.0 -crayon:cpu:standard,unicode,10,28951524.000394344,1390934.9083544528,53.00248129999261,1.3924670036990465,56.75235230000908,2.0804969789494345,84.3644909403051,4.053172312201081,216001.0,0.0 -tiktoken:cl100k_base,code,10,1433910.8675132527,50377.42553661942,0.009309600019946629,0.010288662787986066,0.0004852999950344383,4.36578980047418e-05,4.652997392040999,0.16347322204632306,308000.0,0.0 -tiktoken:cl100k_base,english,10,1796095.8496349081,52793.25030888532,79.61717379999413,251.7495307024531,0.0009670000054029515,0.001478137342961715,9.053785018567714,0.26612095274605746,140001.0,0.0 -tiktoken:cl100k_base,mixed,10,1694892.820124155,17249.8561346943,0.007002899999974943,0.0024695621825708276,0.00046349999536232644,7.491810746504971e-05,5.509475885073746,0.056072965361979525,410000.0,0.0 -tiktoken:cl100k_base,unicode,10,2401549.827809613,288884.55917785416,0.0068406999957915104,0.001325396877696408,0.00044320001961750677,3.8574022529242686e-05,4.939854959582487,0.5942195351837,306000.0,0.0 -tiktoken:o200k_base,code,10,873023.1351143563,99348.24078521204,0.006607900024846458,0.00213868227735025,0.0004508999893459986,3.718258233207919e-05,2.796614003792315,0.31824893322633546,312000.0,0.0 -tiktoken:o200k_base,english,10,943924.5584751449,10877.634765897466,112.38300849999519,355.36493424763796,0.0012451999992890705,0.0023487978392401484,4.758148084311633,0.05483213309632247,140001.0,0.0 -tiktoken:o200k_base,mixed,10,960701.105688441,11606.046915111614,0.006605300001183423,0.0021411037255844156,0.0004602999979397282,8.956942613561185e-05,3.437272129953874,0.041525029339549595,372500.0,0.0 -tiktoken:o200k_base,unicode,10,1218811.126541087,133843.5003360904,0.0061763999866570884,0.0007053232757927612,0.00047830000653448224,4.933797594218068e-05,3.6530964420732652,0.4011640558779565,210000.0,0.0 -tiktoken:p50k_base,code,10,1874283.4220600787,214791.74774445157,0.006904799991502841,0.002517008202014823,0.00047970000025543413,6.630576200775165e-05,4.460128208158045,0.5111279978892581,420000.0,0.0 -tiktoken:p50k_base,english,10,2079144.465738733,132466.7738651459,87.16504709999242,275.62101757793516,0.0012856999774157885,0.002490739036019967,10.189455219298377,0.6491921473403963,144001.0,0.0 -tiktoken:p50k_base,mixed,10,2174162.5156145184,21587.89759563529,0.005877099997064761,0.0006585560836289539,0.0004788999973470709,8.320984769302155e-05,5.5992976305151485,0.055597069210292546,517500.0,0.0 -tiktoken:p50k_base,unicode,10,3512371.3360246257,32909.21781789151,0.007314200007613181,0.00328893151723234,0.0005098000059433616,0.00010494317397068813,5.848609787512716,0.054798640296089184,378000.0,0.0 diff --git a/benchmark_results/20260316_131110/benchmark_summary.json b/benchmark_results/20260316_131110/benchmark_summary.json deleted file mode 100644 index 25b51d3..0000000 --- a/benchmark_results/20260316_131110/benchmark_summary.json +++ /dev/null @@ -1,302 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "code", - "n": 10, - "tokens_per_sec_mean": 32543163.61544498, - "tokens_per_sec_std": 1660418.469066782, - "cold_load_time_ms_mean": 8.423077899999498, - "cold_load_time_ms_std": 0.184571569644575, - "warm_load_time_ms_mean": 7.978830799999059, - "warm_load_time_ms_std": 0.21588091628542938, - "mb_per_sec_mean": 58.49871784489161, - "mb_per_sec_std": 2.984723694173529, - "tokens_produced_mean": 556000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "n": 10, - "tokens_per_sec_mean": 18605171.456859484, - "tokens_per_sec_std": 1031803.966051145, - "cold_load_time_ms_mean": 12.301843200023654, - "cold_load_time_ms_std": 0.9205535110367989, - "warm_load_time_ms_mean": 8.13174449996552, - "warm_load_time_ms_std": 0.31186301379951287, - "mb_per_sec_mean": 91.18008129352394, - "mb_per_sec_std": 5.05665694732621, - "tokens_produced_mean": 144001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "n": 10, - "tokens_per_sec_mean": 26267093.78252945, - "tokens_per_sec_std": 1602078.7282628026, - "cold_load_time_ms_mean": 8.81386069999337, - "cold_load_time_ms_std": 1.2445905519076692, - "warm_load_time_ms_mean": 8.083804200018108, - "warm_load_time_ms_std": 0.28357285227868667, - "mb_per_sec_mean": 54.69957524699704, - "mb_per_sec_std": 3.336228464167218, - "tokens_produced_mean": 640000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "n": 10, - "tokens_per_sec_mean": 42149609.116450116, - "tokens_per_sec_std": 1563828.113027747, - "cold_load_time_ms_mean": 8.860825800007888, - "cold_load_time_ms_std": 1.2318979023066992, - "warm_load_time_ms_mean": 7.961432899998044, - "warm_load_time_ms_std": 0.20027127594089345, - "mb_per_sec_mean": 55.97050585697889, - "mb_per_sec_std": 2.0766088320702214, - "tokens_produced_mean": 474000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "n": 10, - "tokens_per_sec_mean": 18187056.770267364, - "tokens_per_sec_std": 1211012.2796866265, - "cold_load_time_ms_mean": 55.81563290000986, - "cold_load_time_ms_std": 1.5913846515152064, - "warm_load_time_ms_mean": 56.87761500000761, - "warm_load_time_ms_std": 2.2221049765449994, - "mb_per_sec_mean": 58.25982795385274, - "mb_per_sec_std": 3.879317470427001, - "tokens_produced_mean": 312000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "n": 10, - "tokens_per_sec_mean": 17873251.748225935, - "tokens_per_sec_std": 793733.3382264881, - "cold_load_time_ms_mean": 57.222686999995176, - "cold_load_time_ms_std": 0.9429351112404759, - "warm_load_time_ms_mean": 56.375195000009626, - "warm_load_time_ms_std": 0.8928807337267614, - "mb_per_sec_mean": 92.74559191314898, - "mb_per_sec_std": 4.11873951712912, - "tokens_produced_mean": 136001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "n": 10, - "tokens_per_sec_mean": 16543894.171191728, - "tokens_per_sec_std": 597306.6951312533, - "cold_load_time_ms_mean": 53.648192899987635, - "cold_load_time_ms_std": 1.2538254564721554, - "warm_load_time_ms_mean": 57.10128319998091, - "warm_load_time_ms_std": 1.90463315356463, - "mb_per_sec_mean": 59.19188950569539, - "mb_per_sec_std": 2.1370852311656443, - "tokens_produced_mean": 372501.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "n": 10, - "tokens_per_sec_mean": 28951524.000394344, - "tokens_per_sec_std": 1390934.9083544528, - "cold_load_time_ms_mean": 53.00248129999261, - "cold_load_time_ms_std": 1.3924670036990465, - "warm_load_time_ms_mean": 56.75235230000908, - "warm_load_time_ms_std": 2.0804969789494345, - "mb_per_sec_mean": 84.3644909403051, - "mb_per_sec_std": 4.053172312201081, - "tokens_produced_mean": 216001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "n": 10, - "tokens_per_sec_mean": 1433910.8675132527, - "tokens_per_sec_std": 50377.42553661942, - "cold_load_time_ms_mean": 0.009309600019946629, - "cold_load_time_ms_std": 0.010288662787986066, - "warm_load_time_ms_mean": 0.0004852999950344383, - "warm_load_time_ms_std": 4.36578980047418e-05, - "mb_per_sec_mean": 4.652997392040999, - "mb_per_sec_std": 0.16347322204632306, - "tokens_produced_mean": 308000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "n": 10, - "tokens_per_sec_mean": 1796095.8496349081, - "tokens_per_sec_std": 52793.25030888532, - "cold_load_time_ms_mean": 79.61717379999413, - "cold_load_time_ms_std": 251.7495307024531, - "warm_load_time_ms_mean": 0.0009670000054029515, - "warm_load_time_ms_std": 0.001478137342961715, - "mb_per_sec_mean": 9.053785018567714, - "mb_per_sec_std": 0.26612095274605746, - "tokens_produced_mean": 140001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "n": 10, - "tokens_per_sec_mean": 1694892.820124155, - "tokens_per_sec_std": 17249.8561346943, - "cold_load_time_ms_mean": 0.007002899999974943, - "cold_load_time_ms_std": 0.0024695621825708276, - "warm_load_time_ms_mean": 0.00046349999536232644, - "warm_load_time_ms_std": 7.491810746504971e-05, - "mb_per_sec_mean": 5.509475885073746, - "mb_per_sec_std": 0.056072965361979525, - "tokens_produced_mean": 410000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "n": 10, - "tokens_per_sec_mean": 2401549.827809613, - "tokens_per_sec_std": 288884.55917785416, - "cold_load_time_ms_mean": 0.0068406999957915104, - "cold_load_time_ms_std": 0.001325396877696408, - "warm_load_time_ms_mean": 0.00044320001961750677, - "warm_load_time_ms_std": 3.8574022529242686e-05, - "mb_per_sec_mean": 4.939854959582487, - "mb_per_sec_std": 0.5942195351837, - "tokens_produced_mean": 306000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "n": 10, - "tokens_per_sec_mean": 873023.1351143563, - "tokens_per_sec_std": 99348.24078521204, - "cold_load_time_ms_mean": 0.006607900024846458, - "cold_load_time_ms_std": 0.00213868227735025, - "warm_load_time_ms_mean": 0.0004508999893459986, - "warm_load_time_ms_std": 3.718258233207919e-05, - "mb_per_sec_mean": 2.796614003792315, - "mb_per_sec_std": 0.31824893322633546, - "tokens_produced_mean": 312000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "n": 10, - "tokens_per_sec_mean": 943924.5584751449, - "tokens_per_sec_std": 10877.634765897466, - "cold_load_time_ms_mean": 112.38300849999519, - "cold_load_time_ms_std": 355.36493424763796, - "warm_load_time_ms_mean": 0.0012451999992890705, - "warm_load_time_ms_std": 0.0023487978392401484, - "mb_per_sec_mean": 4.758148084311633, - "mb_per_sec_std": 0.05483213309632247, - "tokens_produced_mean": 140001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "n": 10, - "tokens_per_sec_mean": 960701.105688441, - "tokens_per_sec_std": 11606.046915111614, - "cold_load_time_ms_mean": 0.006605300001183423, - "cold_load_time_ms_std": 0.0021411037255844156, - "warm_load_time_ms_mean": 0.0004602999979397282, - "warm_load_time_ms_std": 8.956942613561185e-05, - "mb_per_sec_mean": 3.437272129953874, - "mb_per_sec_std": 0.041525029339549595, - "tokens_produced_mean": 372500.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "n": 10, - "tokens_per_sec_mean": 1218811.126541087, - "tokens_per_sec_std": 133843.5003360904, - "cold_load_time_ms_mean": 0.0061763999866570884, - "cold_load_time_ms_std": 0.0007053232757927612, - "warm_load_time_ms_mean": 0.00047830000653448224, - "warm_load_time_ms_std": 4.933797594218068e-05, - "mb_per_sec_mean": 3.6530964420732652, - "mb_per_sec_std": 0.4011640558779565, - "tokens_produced_mean": 210000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "n": 10, - "tokens_per_sec_mean": 1874283.4220600787, - "tokens_per_sec_std": 214791.74774445157, - "cold_load_time_ms_mean": 0.006904799991502841, - "cold_load_time_ms_std": 0.002517008202014823, - "warm_load_time_ms_mean": 0.00047970000025543413, - "warm_load_time_ms_std": 6.630576200775165e-05, - "mb_per_sec_mean": 4.460128208158045, - "mb_per_sec_std": 0.5111279978892581, - "tokens_produced_mean": 420000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "n": 10, - "tokens_per_sec_mean": 2079144.465738733, - "tokens_per_sec_std": 132466.7738651459, - "cold_load_time_ms_mean": 87.16504709999242, - "cold_load_time_ms_std": 275.62101757793516, - "warm_load_time_ms_mean": 0.0012856999774157885, - "warm_load_time_ms_std": 0.002490739036019967, - "mb_per_sec_mean": 10.189455219298377, - "mb_per_sec_std": 0.6491921473403963, - "tokens_produced_mean": 144001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "n": 10, - "tokens_per_sec_mean": 2174162.5156145184, - "tokens_per_sec_std": 21587.89759563529, - "cold_load_time_ms_mean": 0.005877099997064761, - "cold_load_time_ms_std": 0.0006585560836289539, - "warm_load_time_ms_mean": 0.0004788999973470709, - "warm_load_time_ms_std": 8.320984769302155e-05, - "mb_per_sec_mean": 5.5992976305151485, - "mb_per_sec_std": 0.055597069210292546, - "tokens_produced_mean": 517500.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "n": 10, - "tokens_per_sec_mean": 3512371.3360246257, - "tokens_per_sec_std": 32909.21781789151, - "cold_load_time_ms_mean": 0.007314200007613181, - "cold_load_time_ms_std": 0.00328893151723234, - "warm_load_time_ms_mean": 0.0005098000059433616, - "warm_load_time_ms_std": 0.00010494317397068813, - "mb_per_sec_mean": 5.848609787512716, - "mb_per_sec_std": 0.054798640296089184, - "tokens_produced_mean": 378000.0, - "tokens_produced_std": 0.0 - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_131110/mb_per_sec.png b/benchmark_results/20260316_131110/mb_per_sec.png deleted file mode 100644 index fd67a16..0000000 Binary files a/benchmark_results/20260316_131110/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_131110/metadata.json b/benchmark_results/20260316_131110/metadata.json deleted file mode 100644 index 79e2d8b..0000000 --- a/benchmark_results/20260316_131110/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "timestamp": "2026-03-16T13:11:11", - "cwd": "/app", - "device_arg": "cpu", - "platform": "Linux-6.8.0-x86_64-with-glibc2.39", - "python": "3.12.13 (main, Mar 6 2026, 16:37:31) [GCC 13.3.0]", - "processor": "x86_64", - "cpu_count_logical": 4, - "ram_total_bytes": null, - "nvidia_smi": null, - "rocm_smi": null, - "versions": { - "tiktoken": "0.12.0", - "transformers": "5.3.0", - "matplotlib": "3.10.8" - }, - "backends": { - "torch": null - } -} \ No newline at end of file diff --git a/benchmark_results/20260316_131110/tokens_per_sec.png b/benchmark_results/20260316_131110/tokens_per_sec.png deleted file mode 100644 index 8eeafea..0000000 Binary files a/benchmark_results/20260316_131110/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_131800/benchmark_results.csv b/benchmark_results/20260316_131800/benchmark_results.csv deleted file mode 100644 index 64b8a57..0000000 --- a/benchmark_results/20260316_131800/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,cold_load_time_ms,warm_load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,13.217154000017217,7.726877999971293,144001,740000,7.822760999943057,18407950.85022388,90.21354406018055, -crayon:cpu:lite,code,OK,8.033628999896791,7.926046000079623,556000,1048000,16.766286000006403,33161786.695025224,59.610738096282525, -crayon:cpu:lite,unicode,OK,12.237848000040685,8.257507999928748,474000,660000,10.7920230000218,43921329.67090994,58.32317525887904, -crayon:cpu:lite,mixed,OK,11.667987000009816,7.843707999995786,640000,1397500,26.027087999977994,24589765.86241769,51.206645060671455, -crayon:cpu:standard,english,OK,56.99509800001579,55.52259799992498,136001,740000,7.927816000005805,17154913.78708845,89.01808444344675, -crayon:cpu:standard,code,OK,58.08247499999197,60.035434999917925,312000,1048000,16.677758000014364,18707550.499277618,59.92716068867825, -crayon:cpu:standard,unicode,OK,56.545544999949016,59.173324000084904,216001,660000,7.390334999968218,29227497.80638211,85.1686762279155, -crayon:cpu:standard,mixed,OK,55.16599899999619,58.29951300006542,372501,1397500,21.415187000002334,17394244.56111261,62.23433198027125, -tiktoken:p50k_base,english,OK,115.26536799999576,0.008334999961334688,144001,740000,71.01593200002299,2027728.0878318036,9.937474229591137, -tiktoken:p50k_base,code,OK,0.006592000090677175,0.0005709999868486193,420000,1048000,341.56042799997977,1229650.6432531606,2.926131371383014, -tiktoken:p50k_base,unicode,OK,0.05960099997537327,0.0006080000503061456,378000,660000,166.30912400000852,2272875.9006630364,3.7846693776589957, -tiktoken:p50k_base,mixed,OK,0.013824000006934511,0.0006089999260439072,517500,1397500,364.80763299994123,1418555.844745944,3.653322289936705, -tiktoken:cl100k_base,english,OK,403.35951000008663,0.007544999903075222,140001,740000,116.80072499996186,1198631.2584964323,6.04207717153173, -tiktoken:cl100k_base,code,OK,0.017705999994177546,0.00047600008201698074,308000,1048000,335.9257010000647,916869.4121440285,2.975213508875159, -tiktoken:cl100k_base,unicode,OK,0.006681000058961217,0.0004099999841855606,306000,660000,180.41763400003674,1696064.80927434,3.4887113574940063, -tiktoken:cl100k_base,mixed,OK,0.005176000058781938,0.0004449999551070505,410000,1397500,384.37863700005437,1066656.573840606,3.4673099097792623, -tiktoken:o200k_base,english,OK,819.557422999992,0.014037000028110924,140001,740000,145.22421899994242,964033.4164927099,4.859513096372065, -tiktoken:o200k_base,code,OK,0.00572299995837966,0.00048700007937441114,312000,1048000,337.422953999976,924655.5289182317,2.9620115399553435, -tiktoken:o200k_base,unicode,OK,0.007110000069587841,0.0004639999815481133,210000,660000,165.6797099999494,1267505.840033545,3.7990472631097507, -tiktoken:o200k_base,mixed,OK,0.0056989999848156,0.0008229999366449192,372500,1397500,377.42907800009107,986940.3861880248,3.5311530956748722, diff --git a/benchmark_results/20260316_131800/benchmark_results.json b/benchmark_results/20260316_131800/benchmark_results.json deleted file mode 100644 index 16243d8..0000000 --- a/benchmark_results/20260316_131800/benchmark_results.json +++ /dev/null @@ -1,262 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "cold_load_time_ms": 13.217154000017217, - "warm_load_time_ms": 7.726877999971293, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 7.822760999943057, - "tokens_per_sec": 18407950.85022388, - "mb_per_sec": 90.21354406018055, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "cold_load_time_ms": 8.033628999896791, - "warm_load_time_ms": 7.926046000079623, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 16.766286000006403, - "tokens_per_sec": 33161786.695025224, - "mb_per_sec": 59.610738096282525, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 12.237848000040685, - "warm_load_time_ms": 8.257507999928748, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 10.7920230000218, - "tokens_per_sec": 43921329.67090994, - "mb_per_sec": 58.32317525887904, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 11.667987000009816, - "warm_load_time_ms": 7.843707999995786, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 26.027087999977994, - "tokens_per_sec": 24589765.86241769, - "mb_per_sec": 51.206645060671455, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "cold_load_time_ms": 56.99509800001579, - "warm_load_time_ms": 55.52259799992498, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 7.927816000005805, - "tokens_per_sec": 17154913.78708845, - "mb_per_sec": 89.01808444344675, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "cold_load_time_ms": 58.08247499999197, - "warm_load_time_ms": 60.035434999917925, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 16.677758000014364, - "tokens_per_sec": 18707550.499277618, - "mb_per_sec": 59.92716068867825, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 56.545544999949016, - "warm_load_time_ms": 59.173324000084904, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 7.390334999968218, - "tokens_per_sec": 29227497.80638211, - "mb_per_sec": 85.1686762279155, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 55.16599899999619, - "warm_load_time_ms": 58.29951300006542, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 21.415187000002334, - "tokens_per_sec": 17394244.56111261, - "mb_per_sec": 62.23433198027125, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 115.26536799999576, - "warm_load_time_ms": 0.008334999961334688, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 71.01593200002299, - "tokens_per_sec": 2027728.0878318036, - "mb_per_sec": 9.937474229591137, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.006592000090677175, - "warm_load_time_ms": 0.0005709999868486193, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 341.56042799997977, - "tokens_per_sec": 1229650.6432531606, - "mb_per_sec": 2.926131371383014, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.05960099997537327, - "warm_load_time_ms": 0.0006080000503061456, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 166.30912400000852, - "tokens_per_sec": 2272875.9006630364, - "mb_per_sec": 3.7846693776589957, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.013824000006934511, - "warm_load_time_ms": 0.0006089999260439072, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 364.80763299994123, - "tokens_per_sec": 1418555.844745944, - "mb_per_sec": 3.653322289936705, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 403.35951000008663, - "warm_load_time_ms": 0.007544999903075222, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 116.80072499996186, - "tokens_per_sec": 1198631.2584964323, - "mb_per_sec": 6.04207717153173, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.017705999994177546, - "warm_load_time_ms": 0.00047600008201698074, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 335.9257010000647, - "tokens_per_sec": 916869.4121440285, - "mb_per_sec": 2.975213508875159, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.006681000058961217, - "warm_load_time_ms": 0.0004099999841855606, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 180.41763400003674, - "tokens_per_sec": 1696064.80927434, - "mb_per_sec": 3.4887113574940063, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.005176000058781938, - "warm_load_time_ms": 0.0004449999551070505, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 384.37863700005437, - "tokens_per_sec": 1066656.573840606, - "mb_per_sec": 3.4673099097792623, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "cold_load_time_ms": 819.557422999992, - "warm_load_time_ms": 0.014037000028110924, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 145.22421899994242, - "tokens_per_sec": 964033.4164927099, - "mb_per_sec": 4.859513096372065, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "cold_load_time_ms": 0.00572299995837966, - "warm_load_time_ms": 0.00048700007937441114, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 337.422953999976, - "tokens_per_sec": 924655.5289182317, - "mb_per_sec": 2.9620115399553435, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "cold_load_time_ms": 0.007110000069587841, - "warm_load_time_ms": 0.0004639999815481133, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 165.6797099999494, - "tokens_per_sec": 1267505.840033545, - "mb_per_sec": 3.7990472631097507, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "cold_load_time_ms": 0.0056989999848156, - "warm_load_time_ms": 0.0008229999366449192, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 377.42907800009107, - "tokens_per_sec": 986940.3861880248, - "mb_per_sec": 3.5311530956748722, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_131800/benchmark_summary.csv b/benchmark_results/20260316_131800/benchmark_summary.csv deleted file mode 100644 index f84e0b5..0000000 --- a/benchmark_results/20260316_131800/benchmark_summary.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,n,tokens_per_sec_mean,tokens_per_sec_std,cold_load_time_ms_mean,cold_load_time_ms_std,warm_load_time_ms_mean,warm_load_time_ms_std,mb_per_sec_mean,mb_per_sec_std,tokens_produced_mean,tokens_produced_std -crayon:cpu:lite,code,1,33161786.695025224,0.0,8.033628999896791,0.0,7.926046000079623,0.0,59.610738096282525,0.0,556000.0,0.0 -crayon:cpu:lite,english,1,18407950.85022388,0.0,13.217154000017217,0.0,7.726877999971293,0.0,90.21354406018055,0.0,144001.0,0.0 -crayon:cpu:lite,mixed,1,24589765.86241769,0.0,11.667987000009816,0.0,7.843707999995786,0.0,51.206645060671455,0.0,640000.0,0.0 -crayon:cpu:lite,unicode,1,43921329.67090994,0.0,12.237848000040685,0.0,8.257507999928748,0.0,58.32317525887904,0.0,474000.0,0.0 -crayon:cpu:standard,code,1,18707550.499277618,0.0,58.08247499999197,0.0,60.035434999917925,0.0,59.92716068867825,0.0,312000.0,0.0 -crayon:cpu:standard,english,1,17154913.78708845,0.0,56.99509800001579,0.0,55.52259799992498,0.0,89.01808444344675,0.0,136001.0,0.0 -crayon:cpu:standard,mixed,1,17394244.56111261,0.0,55.16599899999619,0.0,58.29951300006542,0.0,62.23433198027125,0.0,372501.0,0.0 -crayon:cpu:standard,unicode,1,29227497.80638211,0.0,56.545544999949016,0.0,59.173324000084904,0.0,85.1686762279155,0.0,216001.0,0.0 -tiktoken:cl100k_base,code,1,916869.4121440285,0.0,0.017705999994177546,0.0,0.00047600008201698074,0.0,2.975213508875159,0.0,308000.0,0.0 -tiktoken:cl100k_base,english,1,1198631.2584964323,0.0,403.35951000008663,0.0,0.007544999903075222,0.0,6.04207717153173,0.0,140001.0,0.0 -tiktoken:cl100k_base,mixed,1,1066656.573840606,0.0,0.005176000058781938,0.0,0.0004449999551070505,0.0,3.4673099097792623,0.0,410000.0,0.0 -tiktoken:cl100k_base,unicode,1,1696064.80927434,0.0,0.006681000058961217,0.0,0.0004099999841855606,0.0,3.4887113574940063,0.0,306000.0,0.0 -tiktoken:o200k_base,code,1,924655.5289182317,0.0,0.00572299995837966,0.0,0.00048700007937441114,0.0,2.9620115399553435,0.0,312000.0,0.0 -tiktoken:o200k_base,english,1,964033.4164927099,0.0,819.557422999992,0.0,0.014037000028110924,0.0,4.859513096372065,0.0,140001.0,0.0 -tiktoken:o200k_base,mixed,1,986940.3861880248,0.0,0.0056989999848156,0.0,0.0008229999366449192,0.0,3.5311530956748722,0.0,372500.0,0.0 -tiktoken:o200k_base,unicode,1,1267505.840033545,0.0,0.007110000069587841,0.0,0.0004639999815481133,0.0,3.7990472631097507,0.0,210000.0,0.0 -tiktoken:p50k_base,code,1,1229650.6432531606,0.0,0.006592000090677175,0.0,0.0005709999868486193,0.0,2.926131371383014,0.0,420000.0,0.0 -tiktoken:p50k_base,english,1,2027728.0878318036,0.0,115.26536799999576,0.0,0.008334999961334688,0.0,9.937474229591137,0.0,144001.0,0.0 -tiktoken:p50k_base,mixed,1,1418555.844745944,0.0,0.013824000006934511,0.0,0.0006089999260439072,0.0,3.653322289936705,0.0,517500.0,0.0 -tiktoken:p50k_base,unicode,1,2272875.9006630364,0.0,0.05960099997537327,0.0,0.0006080000503061456,0.0,3.7846693776589957,0.0,378000.0,0.0 diff --git a/benchmark_results/20260316_131800/benchmark_summary.json b/benchmark_results/20260316_131800/benchmark_summary.json deleted file mode 100644 index 82bc3cb..0000000 --- a/benchmark_results/20260316_131800/benchmark_summary.json +++ /dev/null @@ -1,302 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "code", - "n": 1, - "tokens_per_sec_mean": 33161786.695025224, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 8.033628999896791, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 7.926046000079623, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 59.610738096282525, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 556000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:lite", - "case": "english", - "n": 1, - "tokens_per_sec_mean": 18407950.85022388, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 13.217154000017217, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 7.726877999971293, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 90.21354406018055, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 144001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "n": 1, - "tokens_per_sec_mean": 24589765.86241769, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 11.667987000009816, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 7.843707999995786, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 51.206645060671455, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 640000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "n": 1, - "tokens_per_sec_mean": 43921329.67090994, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 12.237848000040685, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 8.257507999928748, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 58.32317525887904, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 474000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "n": 1, - "tokens_per_sec_mean": 18707550.499277618, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 58.08247499999197, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 60.035434999917925, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 59.92716068867825, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 312000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "n": 1, - "tokens_per_sec_mean": 17154913.78708845, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 56.99509800001579, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 55.52259799992498, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 89.01808444344675, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 136001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "n": 1, - "tokens_per_sec_mean": 17394244.56111261, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 55.16599899999619, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 58.29951300006542, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 62.23433198027125, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 372501.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "n": 1, - "tokens_per_sec_mean": 29227497.80638211, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 56.545544999949016, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 59.173324000084904, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 85.1686762279155, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 216001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "n": 1, - "tokens_per_sec_mean": 916869.4121440285, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.017705999994177546, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.00047600008201698074, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 2.975213508875159, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 308000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "n": 1, - "tokens_per_sec_mean": 1198631.2584964323, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 403.35951000008663, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.007544999903075222, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 6.04207717153173, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 140001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "n": 1, - "tokens_per_sec_mean": 1066656.573840606, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.005176000058781938, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.0004449999551070505, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 3.4673099097792623, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 410000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "n": 1, - "tokens_per_sec_mean": 1696064.80927434, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.006681000058961217, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.0004099999841855606, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 3.4887113574940063, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 306000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "n": 1, - "tokens_per_sec_mean": 924655.5289182317, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.00572299995837966, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.00048700007937441114, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 2.9620115399553435, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 312000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "n": 1, - "tokens_per_sec_mean": 964033.4164927099, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 819.557422999992, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.014037000028110924, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 4.859513096372065, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 140001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "n": 1, - "tokens_per_sec_mean": 986940.3861880248, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.0056989999848156, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.0008229999366449192, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 3.5311530956748722, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 372500.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "n": 1, - "tokens_per_sec_mean": 1267505.840033545, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.007110000069587841, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.0004639999815481133, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 3.7990472631097507, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 210000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "n": 1, - "tokens_per_sec_mean": 1229650.6432531606, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.006592000090677175, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.0005709999868486193, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 2.926131371383014, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 420000.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "n": 1, - "tokens_per_sec_mean": 2027728.0878318036, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 115.26536799999576, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.008334999961334688, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 9.937474229591137, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 144001.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "n": 1, - "tokens_per_sec_mean": 1418555.844745944, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.013824000006934511, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.0006089999260439072, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 3.653322289936705, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 517500.0, - "tokens_produced_std": 0.0 - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "n": 1, - "tokens_per_sec_mean": 2272875.9006630364, - "tokens_per_sec_std": 0.0, - "cold_load_time_ms_mean": 0.05960099997537327, - "cold_load_time_ms_std": 0.0, - "warm_load_time_ms_mean": 0.0006080000503061456, - "warm_load_time_ms_std": 0.0, - "mb_per_sec_mean": 3.7846693776589957, - "mb_per_sec_std": 0.0, - "tokens_produced_mean": 378000.0, - "tokens_produced_std": 0.0 - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_131800/load_time_ms.png b/benchmark_results/20260316_131800/load_time_ms.png deleted file mode 100644 index 7c3ffe7..0000000 Binary files a/benchmark_results/20260316_131800/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260316_131800/mb_per_sec.png b/benchmark_results/20260316_131800/mb_per_sec.png deleted file mode 100644 index 9aa5e61..0000000 Binary files a/benchmark_results/20260316_131800/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_131800/metadata.json b/benchmark_results/20260316_131800/metadata.json deleted file mode 100644 index 018f1e8..0000000 --- a/benchmark_results/20260316_131800/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "timestamp": "2026-03-16T13:18:00", - "cwd": "/app", - "device_arg": "cpu", - "platform": "Linux-6.8.0-x86_64-with-glibc2.39", - "python": "3.12.13 (main, Mar 6 2026, 16:37:31) [GCC 13.3.0]", - "processor": "x86_64", - "cpu_count_logical": 4, - "ram_total_bytes": null, - "nvidia_smi": null, - "rocm_smi": null, - "versions": { - "tiktoken": "0.12.0", - "transformers": "5.3.0", - "matplotlib": "3.10.8" - }, - "backends": { - "torch": null - } -} \ No newline at end of file diff --git a/benchmark_results/20260316_131800/tokens_per_sec.png b/benchmark_results/20260316_131800/tokens_per_sec.png deleted file mode 100644 index 9a48b68..0000000 Binary files a/benchmark_results/20260316_131800/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_131800/tokens_produced.png b/benchmark_results/20260316_131800/tokens_produced.png deleted file mode 100644 index f8b4b91..0000000 Binary files a/benchmark_results/20260316_131800/tokens_produced.png and /dev/null differ diff --git a/benchmark_results/20260316_132855/benchmark_results.csv b/benchmark_results/20260316_132855/benchmark_results.csv deleted file mode 100644 index 74776e2..0000000 --- a/benchmark_results/20260316_132855/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,105.03960028290749,144001,740000,25.129109993577003,5730445.687762385,28.08372418764559, -crayon:cpu:lite,code,OK,21.776700392365456,556000,1048000,40.840200148522854,13614037.100161223,24.47222785292591, -crayon:cpu:lite,unicode,OK,20.329000428318977,474000,660000,34.67500973492861,13669787.077883163,18.15212320457106, -crayon:cpu:lite,mixed,OK,19.497700035572052,640000,1397500,50.04436010494828,12788653.87943522,26.631569559143067, -crayon:cpu:standard,english,OK,114.62089978158474,136001,740000,13.758950028568506,9884547.855585873,51.2916314599079, -crayon:cpu:standard,code,OK,85.3595994412899,312000,1048000,26.4725299552083,11785802.132546684,37.754256404084806, -crayon:cpu:standard,unicode,OK,91.49810019880533,216001,660000,14.188029989600182,15224171.372511098,44.36310391854918, -crayon:cpu:standard,mixed,OK,95.99119983613491,372501,1397500,37.69492004066706,9881994.698440222,35.35648452735515, -tiktoken:p50k_base,english,OK,272.5568003952503,144001,740000,236.03122998028994,610092.9949482743,2.989939061028309, -tiktoken:p50k_base,code,OK,0.01019984483718872,420000,1048000,1071.3457399979234,392030.30760248704,0.9328927593399375, -tiktoken:p50k_base,unicode,OK,0.006799586117267609,378000,660000,763.4834598749876,495099.1342522254,0.8244121607181729, -tiktoken:p50k_base,mixed,OK,0.005899928510189056,517500,1397500,1304.1077699512243,396823.0325162125,1.0219706437510006, -tiktoken:cl100k_base,english,OK,553.0430004000664,140001,740000,314.7866601124406,444748.833860978,2.2418961270104165, -tiktoken:cl100k_base,code,OK,0.00690016895532608,308000,1048000,591.5066200308502,520704.2314825423,1.68966948086164, -tiktoken:cl100k_base,unicode,OK,0.0056996941566467285,306000,660000,372.45238004252315,821581.5400751736,1.6899477156147134, -tiktoken:cl100k_base,mixed,OK,0.010100193321704865,410000,1397500,674.7905101627111,607595.9780482647,1.9750720217691977, -tiktoken:o200k_base,english,OK,791.211100295186,140001,740000,375.6954999640584,372644.8680204939,1.8784334499831352, -tiktoken:o200k_base,code,OK,0.0058002769947052,312000,1048000,806.401920132339,386903.8406416462,1.2393952179946814, -tiktoken:o200k_base,unicode,OK,0.0064997002482414246,210000,660000,440.8437602221966,476359.2432252066,1.427773523460735, -tiktoken:o200k_base,mixed,OK,0.0058002769947052,372500,1397500,885.7569799758494,420544.2445513175,1.5046563417587437, diff --git a/benchmark_results/20260316_132855/benchmark_results.json b/benchmark_results/20260316_132855/benchmark_results.json deleted file mode 100644 index 6eec272..0000000 --- a/benchmark_results/20260316_132855/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 105.03960028290749, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 25.129109993577003, - "tokens_per_sec": 5730445.687762385, - "mb_per_sec": 28.08372418764559, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 21.776700392365456, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 40.840200148522854, - "tokens_per_sec": 13614037.100161223, - "mb_per_sec": 24.47222785292591, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 20.329000428318977, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 34.67500973492861, - "tokens_per_sec": 13669787.077883163, - "mb_per_sec": 18.15212320457106, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 19.497700035572052, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 50.04436010494828, - "tokens_per_sec": 12788653.87943522, - "mb_per_sec": 26.631569559143067, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 114.62089978158474, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 13.758950028568506, - "tokens_per_sec": 9884547.855585873, - "mb_per_sec": 51.2916314599079, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 85.3595994412899, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 26.4725299552083, - "tokens_per_sec": 11785802.132546684, - "mb_per_sec": 37.754256404084806, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 91.49810019880533, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 14.188029989600182, - "tokens_per_sec": 15224171.372511098, - "mb_per_sec": 44.36310391854918, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 95.99119983613491, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 37.69492004066706, - "tokens_per_sec": 9881994.698440222, - "mb_per_sec": 35.35648452735515, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 272.5568003952503, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 236.03122998028994, - "tokens_per_sec": 610092.9949482743, - "mb_per_sec": 2.989939061028309, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.01019984483718872, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 1071.3457399979234, - "tokens_per_sec": 392030.30760248704, - "mb_per_sec": 0.9328927593399375, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.006799586117267609, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 763.4834598749876, - "tokens_per_sec": 495099.1342522254, - "mb_per_sec": 0.8244121607181729, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005899928510189056, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 1304.1077699512243, - "tokens_per_sec": 396823.0325162125, - "mb_per_sec": 1.0219706437510006, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 553.0430004000664, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 314.7866601124406, - "tokens_per_sec": 444748.833860978, - "mb_per_sec": 2.2418961270104165, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.00690016895532608, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 591.5066200308502, - "tokens_per_sec": 520704.2314825423, - "mb_per_sec": 1.68966948086164, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.0056996941566467285, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 372.45238004252315, - "tokens_per_sec": 821581.5400751736, - "mb_per_sec": 1.6899477156147134, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.010100193321704865, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 674.7905101627111, - "tokens_per_sec": 607595.9780482647, - "mb_per_sec": 1.9750720217691977, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 791.211100295186, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 375.6954999640584, - "tokens_per_sec": 372644.8680204939, - "mb_per_sec": 1.8784334499831352, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.0058002769947052, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 806.401920132339, - "tokens_per_sec": 386903.8406416462, - "mb_per_sec": 1.2393952179946814, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.0064997002482414246, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 440.8437602221966, - "tokens_per_sec": 476359.2432252066, - "mb_per_sec": 1.427773523460735, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.0058002769947052, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 885.7569799758494, - "tokens_per_sec": 420544.2445513175, - "mb_per_sec": 1.5046563417587437, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_132855/load_time_ms.png b/benchmark_results/20260316_132855/load_time_ms.png deleted file mode 100644 index b51ccca..0000000 Binary files a/benchmark_results/20260316_132855/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260316_132855/mb_per_sec.png b/benchmark_results/20260316_132855/mb_per_sec.png deleted file mode 100644 index 914fd88..0000000 Binary files a/benchmark_results/20260316_132855/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_132855/tokens_per_sec.png b/benchmark_results/20260316_132855/tokens_per_sec.png deleted file mode 100644 index 62ca9e8..0000000 Binary files a/benchmark_results/20260316_132855/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_132855/tokens_produced.png b/benchmark_results/20260316_132855/tokens_produced.png deleted file mode 100644 index ea990ae..0000000 Binary files a/benchmark_results/20260316_132855/tokens_produced.png and /dev/null differ diff --git a/benchmark_results/20260316_140602/benchmark_results.csv b/benchmark_results/20260316_140602/benchmark_results.csv deleted file mode 100644 index 72f99ec..0000000 --- a/benchmark_results/20260316_140602/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,61.401099897921085,144001,740000,11.133879888802767,12933586.623726774,63.38482192989702, -crayon:cpu:lite,code,OK,16.328000463545322,556000,1048000,43.03158987313509,12920740.359331101,23.225976231422344, -crayon:cpu:lite,unicode,OK,22.586899809539318,474000,660000,30.570899788290262,15504941.0806534,20.589025942547398, -crayon:cpu:lite,mixed,OK,17.361399717628956,640000,1397500,50.97524989396334,12555112.556217032,26.145234401990923, -crayon:cpu:standard,english,OK,89.92770034819841,136001,740000,14.09686990082264,9647602.691719776,50.06210592178636, -crayon:cpu:standard,code,OK,92.18220040202141,312000,1048000,27.118550147861242,11505039.845377076,36.854871596908495, -crayon:cpu:standard,unicode,OK,79.41350061446428,216001,660000,17.373310029506683,12432921.511971276,36.22942592742055, -crayon:cpu:standard,mixed,OK,85.39990056306124,372501,1397500,40.105349849909544,9288062.599978546,33.23147316169691, -tiktoken:p50k_base,english,OK,224.58230052143335,144001,740000,199.0367698483169,723489.434187166,3.5456714589894287, -tiktoken:p50k_base,code,OK,0.011400319635868073,420000,1048000,659.9957201629877,636367.7629550081,1.5143290373866864, -tiktoken:p50k_base,unicode,OK,0.006799586117267609,378000,660000,315.81203984096646,1196914.4690948122,1.9930368998759032, -tiktoken:p50k_base,mixed,OK,0.005499459803104401,517500,1397500,711.7198899388313,727111.8979750257,1.872590433424979, -tiktoken:cl100k_base,english,OK,413.1496995687485,140001,740000,247.4105198867619,565865.1865897921,2.8524211277015534, -tiktoken:cl100k_base,code,OK,0.019200146198272705,308000,1048000,638.1493899971247,482645.6074828933,1.566170397182787, -tiktoken:cl100k_base,unicode,OK,0.005200505256652832,306000,660000,341.1180700175464,897050.1034561435,1.8451823698338488, -tiktoken:cl100k_base,mixed,OK,0.005000270903110504,410000,1397500,684.4099200330675,599056.1913249164,1.9473123024186174, -tiktoken:o200k_base,english,OK,847.2124002873898,140001,740000,391.4701501838863,357628.7998822821,1.8027402442028486, -tiktoken:o200k_base,code,OK,0.004900619387626648,312000,1048000,808.8262700475752,385744.1474813227,1.2356802945272316, -tiktoken:o200k_base,unicode,OK,0.004900619387626648,210000,660000,382.0475898683071,549669.7415952489,1.6475043044901543, -tiktoken:o200k_base,mixed,OK,0.0048996880650520325,372500,1397500,951.561520062387,391461.81528607616,1.4006029343119664, diff --git a/benchmark_results/20260316_140602/benchmark_results.json b/benchmark_results/20260316_140602/benchmark_results.json deleted file mode 100644 index 428e7ed..0000000 --- a/benchmark_results/20260316_140602/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 61.401099897921085, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 11.133879888802767, - "tokens_per_sec": 12933586.623726774, - "mb_per_sec": 63.38482192989702, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 16.328000463545322, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 43.03158987313509, - "tokens_per_sec": 12920740.359331101, - "mb_per_sec": 23.225976231422344, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 22.586899809539318, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 30.570899788290262, - "tokens_per_sec": 15504941.0806534, - "mb_per_sec": 20.589025942547398, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 17.361399717628956, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 50.97524989396334, - "tokens_per_sec": 12555112.556217032, - "mb_per_sec": 26.145234401990923, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 89.92770034819841, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 14.09686990082264, - "tokens_per_sec": 9647602.691719776, - "mb_per_sec": 50.06210592178636, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 92.18220040202141, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 27.118550147861242, - "tokens_per_sec": 11505039.845377076, - "mb_per_sec": 36.854871596908495, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 79.41350061446428, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 17.373310029506683, - "tokens_per_sec": 12432921.511971276, - "mb_per_sec": 36.22942592742055, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 85.39990056306124, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 40.105349849909544, - "tokens_per_sec": 9288062.599978546, - "mb_per_sec": 33.23147316169691, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 224.58230052143335, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 199.0367698483169, - "tokens_per_sec": 723489.434187166, - "mb_per_sec": 3.5456714589894287, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.011400319635868073, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 659.9957201629877, - "tokens_per_sec": 636367.7629550081, - "mb_per_sec": 1.5143290373866864, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.006799586117267609, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 315.81203984096646, - "tokens_per_sec": 1196914.4690948122, - "mb_per_sec": 1.9930368998759032, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005499459803104401, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 711.7198899388313, - "tokens_per_sec": 727111.8979750257, - "mb_per_sec": 1.872590433424979, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 413.1496995687485, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 247.4105198867619, - "tokens_per_sec": 565865.1865897921, - "mb_per_sec": 2.8524211277015534, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.019200146198272705, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 638.1493899971247, - "tokens_per_sec": 482645.6074828933, - "mb_per_sec": 1.566170397182787, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.005200505256652832, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 341.1180700175464, - "tokens_per_sec": 897050.1034561435, - "mb_per_sec": 1.8451823698338488, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.005000270903110504, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 684.4099200330675, - "tokens_per_sec": 599056.1913249164, - "mb_per_sec": 1.9473123024186174, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 847.2124002873898, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 391.4701501838863, - "tokens_per_sec": 357628.7998822821, - "mb_per_sec": 1.8027402442028486, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.004900619387626648, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 808.8262700475752, - "tokens_per_sec": 385744.1474813227, - "mb_per_sec": 1.2356802945272316, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.004900619387626648, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 382.0475898683071, - "tokens_per_sec": 549669.7415952489, - "mb_per_sec": 1.6475043044901543, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.0048996880650520325, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 951.561520062387, - "tokens_per_sec": 391461.81528607616, - "mb_per_sec": 1.4006029343119664, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_140602/load_time_ms.png b/benchmark_results/20260316_140602/load_time_ms.png deleted file mode 100644 index 8f24801..0000000 Binary files a/benchmark_results/20260316_140602/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260316_140602/mb_per_sec.png b/benchmark_results/20260316_140602/mb_per_sec.png deleted file mode 100644 index eb7caac..0000000 Binary files a/benchmark_results/20260316_140602/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_140602/tokens_per_sec.png b/benchmark_results/20260316_140602/tokens_per_sec.png deleted file mode 100644 index c178c7d..0000000 Binary files a/benchmark_results/20260316_140602/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_140602/tokens_produced.png b/benchmark_results/20260316_140602/tokens_produced.png deleted file mode 100644 index ea990ae..0000000 Binary files a/benchmark_results/20260316_140602/tokens_produced.png and /dev/null differ diff --git a/benchmark_results/20260316_144511/benchmark_results.csv b/benchmark_results/20260316_144511/benchmark_results.csv deleted file mode 100644 index 39111d8..0000000 --- a/benchmark_results/20260316_144511/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,82.65180047601461,144001,740000,39.757420122623444,3621990.550590533,17.750623455042668, -crayon:cpu:lite,code,OK,96.5157002210617,556000,1048000,79.59905993193388,6985007.115353402,12.55606139630785, -crayon:cpu:lite,unicode,OK,29.729800298810005,474000,660000,74.76676013320684,6339715.659144605,8.418514426821774, -crayon:cpu:lite,mixed,OK,39.704300463199615,640000,1397500,105.36483991891146,6074132.514153132,12.648999971939627, -crayon:cpu:standard,english,OK,363.7454994022846,136001,740000,31.185439974069595,4361041.56661197,22.629759103204055, -crayon:cpu:standard,code,OK,246.52619939297438,312000,1048000,60.35767998546362,5169184.767789969,16.558798877532322, -crayon:cpu:standard,unicode,OK,238.10609988868237,216001,660000,19.04196012765169,11343422.554820666,33.05463537412351, -crayon:cpu:standard,mixed,OK,95.19869927316904,372501,1397500,44.21368017792702,8425016.838701548,30.14360830888476, -tiktoken:p50k_base,english,OK,278.6563001573086,144001,740000,185.2792602032423,777210.5730670445,3.8089476035606236, -tiktoken:p50k_base,code,OK,0.01200009137392044,420000,1048000,697.9285599663854,601780.7897419023,1.4320243373360262, -tiktoken:p50k_base,unicode,OK,0.0051995739340782166,378000,660000,466.2848200649023,810663.3193578682,1.3498724851058097, -tiktoken:p50k_base,mixed,OK,0.0074999406933784485,517500,1397500,917.3948403447866,564097.3518071093,1.452765808751268, -tiktoken:cl100k_base,english,OK,599.8977003619075,140001,740000,517.4914799630642,270537.7874240413,1.3637306534805085, -tiktoken:cl100k_base,code,OK,0.006400048732757568,308000,1048000,697.0198402181268,441881.25248144136,1.4338912982462306, -tiktoken:cl100k_base,unicode,OK,0.0048996880650520325,306000,660000,319.7970999404788,956856.707133846,1.9682012405530718, -tiktoken:cl100k_base,mixed,OK,0.0056996941566467285,410000,1397500,713.6479198932648,574512.9896284442,1.8675313414730694, -tiktoken:o200k_base,english,OK,822.6224007084966,140001,740000,382.0364801213145,366459.7683329694,1.847255513181689, -tiktoken:o200k_base,code,OK,0.00509992241859436,312000,1048000,817.2225998714566,381780.9248656063,1.222984635705078, -tiktoken:o200k_base,unicode,OK,0.0048996880650520325,210000,660000,381.83012027293444,549982.803477868,1.648442633017553, -tiktoken:o200k_base,mixed,OK,0.006800517439842224,372500,1397500,950.2225399017334,392013.43302014476,1.402576555714581, diff --git a/benchmark_results/20260316_144511/benchmark_results.json b/benchmark_results/20260316_144511/benchmark_results.json deleted file mode 100644 index ff3b410..0000000 --- a/benchmark_results/20260316_144511/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 82.65180047601461, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 39.757420122623444, - "tokens_per_sec": 3621990.550590533, - "mb_per_sec": 17.750623455042668, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 96.5157002210617, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 79.59905993193388, - "tokens_per_sec": 6985007.115353402, - "mb_per_sec": 12.55606139630785, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 29.729800298810005, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 74.76676013320684, - "tokens_per_sec": 6339715.659144605, - "mb_per_sec": 8.418514426821774, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 39.704300463199615, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 105.36483991891146, - "tokens_per_sec": 6074132.514153132, - "mb_per_sec": 12.648999971939627, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 363.7454994022846, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 31.185439974069595, - "tokens_per_sec": 4361041.56661197, - "mb_per_sec": 22.629759103204055, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 246.52619939297438, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 60.35767998546362, - "tokens_per_sec": 5169184.767789969, - "mb_per_sec": 16.558798877532322, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 238.10609988868237, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 19.04196012765169, - "tokens_per_sec": 11343422.554820666, - "mb_per_sec": 33.05463537412351, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 95.19869927316904, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 44.21368017792702, - "tokens_per_sec": 8425016.838701548, - "mb_per_sec": 30.14360830888476, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 278.6563001573086, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 185.2792602032423, - "tokens_per_sec": 777210.5730670445, - "mb_per_sec": 3.8089476035606236, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.01200009137392044, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 697.9285599663854, - "tokens_per_sec": 601780.7897419023, - "mb_per_sec": 1.4320243373360262, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.0051995739340782166, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 466.2848200649023, - "tokens_per_sec": 810663.3193578682, - "mb_per_sec": 1.3498724851058097, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.0074999406933784485, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 917.3948403447866, - "tokens_per_sec": 564097.3518071093, - "mb_per_sec": 1.452765808751268, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 599.8977003619075, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 517.4914799630642, - "tokens_per_sec": 270537.7874240413, - "mb_per_sec": 1.3637306534805085, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.006400048732757568, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 697.0198402181268, - "tokens_per_sec": 441881.25248144136, - "mb_per_sec": 1.4338912982462306, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.0048996880650520325, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 319.7970999404788, - "tokens_per_sec": 956856.707133846, - "mb_per_sec": 1.9682012405530718, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.0056996941566467285, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 713.6479198932648, - "tokens_per_sec": 574512.9896284442, - "mb_per_sec": 1.8675313414730694, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 822.6224007084966, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 382.0364801213145, - "tokens_per_sec": 366459.7683329694, - "mb_per_sec": 1.847255513181689, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.00509992241859436, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 817.2225998714566, - "tokens_per_sec": 381780.9248656063, - "mb_per_sec": 1.222984635705078, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.0048996880650520325, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 381.83012027293444, - "tokens_per_sec": 549982.803477868, - "mb_per_sec": 1.648442633017553, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.006800517439842224, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 950.2225399017334, - "tokens_per_sec": 392013.43302014476, - "mb_per_sec": 1.402576555714581, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_144511/load_time_ms.png b/benchmark_results/20260316_144511/load_time_ms.png deleted file mode 100644 index 48fb91b..0000000 Binary files a/benchmark_results/20260316_144511/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260316_144511/mb_per_sec.png b/benchmark_results/20260316_144511/mb_per_sec.png deleted file mode 100644 index a1ac10e..0000000 Binary files a/benchmark_results/20260316_144511/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_144511/tokens_per_sec.png b/benchmark_results/20260316_144511/tokens_per_sec.png deleted file mode 100644 index 1893e9d..0000000 Binary files a/benchmark_results/20260316_144511/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_144511/tokens_produced.png b/benchmark_results/20260316_144511/tokens_produced.png deleted file mode 100644 index ea990ae..0000000 Binary files a/benchmark_results/20260316_144511/tokens_produced.png and /dev/null differ diff --git a/benchmark_results/20260316_144732/benchmark_results.csv b/benchmark_results/20260316_144732/benchmark_results.csv deleted file mode 100644 index c286d72..0000000 --- a/benchmark_results/20260316_144732/benchmark_results.csv +++ /dev/null @@ -1,21 +0,0 @@ -impl,case,status,load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes -crayon:cpu:lite,english,OK,22.311399690806866,144001,740000,12.09609005600214,11904755.9445497,58.34273644403331, -crayon:cpu:lite,code,OK,17.9625004529953,556000,1048000,38.264369778335094,14530488.89138641,26.11961700620061, -crayon:cpu:lite,unicode,OK,20.527299493551254,474000,660000,27.283419854938984,17373188.6442452,23.069873651274815, -crayon:cpu:lite,mixed,OK,17.815999686717987,640000,1397500,46.79785007610917,13675841.923488857,28.479083013647315, -crayon:cpu:standard,english,OK,79.29230015724897,136001,740000,11.559089925140142,11765718.657851094,61.053162377925595, -crayon:cpu:standard,code,OK,87.1273996308446,312000,1048000,48.504129983484745,6432441.940639558,20.60547594470934, -crayon:cpu:standard,unicode,OK,141.48390013724566,216001,660000,13.834299985319376,15613439.077453505,45.49742664941888, -crayon:cpu:standard,mixed,OK,89.90879915654659,372501,1397500,35.81156004220247,10401697.093369367,37.21591172255917, -tiktoken:p50k_base,english,OK,207.154200412333,144001,740000,225.98392013460398,637217.9043280068,3.1228726084593714, -tiktoken:p50k_base,code,OK,0.009899958968162537,420000,1048000,643.2205001823604,652964.2632362078,1.5538228077469456, -tiktoken:p50k_base,unicode,OK,0.006600283086299896,378000,660000,318.2939298450947,1187581.554520888,1.9774962379409802, -tiktoken:p50k_base,mixed,OK,0.004800036549568176,517500,1397500,704.3563799932599,734713.2995444039,1.892166941386245, -tiktoken:cl100k_base,english,OK,390.31270053237677,140001,740000,278.1250501982868,503374.2911693411,2.537416150172338, -tiktoken:cl100k_base,code,OK,0.0066999346017837524,308000,1048000,607.443190086633,507043.30055304983,1.6453401732122626, -tiktoken:cl100k_base,unicode,OK,0.00690016895532608,306000,660000,358.5609101690352,853411.4883179637,1.7554201558987488, -tiktoken:cl100k_base,mixed,OK,0.3294004127383232,410000,1397500,696.1764998733997,588931.1116858424,1.9143993763364577, -tiktoken:o200k_base,english,OK,856.5244004130363,140001,740000,376.90996024757624,371444.15050225594,1.87238085636439, -tiktoken:o200k_base,code,OK,0.004699453711509705,312000,1048000,818.470739852637,381198.721967965,1.2211196258203412, -tiktoken:o200k_base,unicode,OK,0.004400499165058136,210000,660000,383.4480899386108,547662.136049812,1.6414869844022293, -tiktoken:o200k_base,mixed,OK,0.004800036549568176,372500,1397500,927.4989400058985,401617.7096629685,1.4369394936119912, diff --git a/benchmark_results/20260316_144732/benchmark_results.json b/benchmark_results/20260316_144732/benchmark_results.json deleted file mode 100644 index 6fa211d..0000000 --- a/benchmark_results/20260316_144732/benchmark_results.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "impl": "crayon:cpu:lite", - "case": "english", - "status": "OK", - "load_time_ms": 22.311399690806866, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 12.09609005600214, - "tokens_per_sec": 11904755.9445497, - "mb_per_sec": 58.34273644403331, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "code", - "status": "OK", - "load_time_ms": 17.9625004529953, - "tokens_produced": 556000, - "bytes_processed": 1048000, - "avg_time_ms": 38.264369778335094, - "tokens_per_sec": 14530488.89138641, - "mb_per_sec": 26.11961700620061, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "unicode", - "status": "OK", - "load_time_ms": 20.527299493551254, - "tokens_produced": 474000, - "bytes_processed": 660000, - "avg_time_ms": 27.283419854938984, - "tokens_per_sec": 17373188.6442452, - "mb_per_sec": 23.069873651274815, - "notes": "" - }, - { - "impl": "crayon:cpu:lite", - "case": "mixed", - "status": "OK", - "load_time_ms": 17.815999686717987, - "tokens_produced": 640000, - "bytes_processed": 1397500, - "avg_time_ms": 46.79785007610917, - "tokens_per_sec": 13675841.923488857, - "mb_per_sec": 28.479083013647315, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "english", - "status": "OK", - "load_time_ms": 79.29230015724897, - "tokens_produced": 136001, - "bytes_processed": 740000, - "avg_time_ms": 11.559089925140142, - "tokens_per_sec": 11765718.657851094, - "mb_per_sec": 61.053162377925595, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "code", - "status": "OK", - "load_time_ms": 87.1273996308446, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 48.504129983484745, - "tokens_per_sec": 6432441.940639558, - "mb_per_sec": 20.60547594470934, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "unicode", - "status": "OK", - "load_time_ms": 141.48390013724566, - "tokens_produced": 216001, - "bytes_processed": 660000, - "avg_time_ms": 13.834299985319376, - "tokens_per_sec": 15613439.077453505, - "mb_per_sec": 45.49742664941888, - "notes": "" - }, - { - "impl": "crayon:cpu:standard", - "case": "mixed", - "status": "OK", - "load_time_ms": 89.90879915654659, - "tokens_produced": 372501, - "bytes_processed": 1397500, - "avg_time_ms": 35.81156004220247, - "tokens_per_sec": 10401697.093369367, - "mb_per_sec": 37.21591172255917, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "english", - "status": "OK", - "load_time_ms": 207.154200412333, - "tokens_produced": 144001, - "bytes_processed": 740000, - "avg_time_ms": 225.98392013460398, - "tokens_per_sec": 637217.9043280068, - "mb_per_sec": 3.1228726084593714, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.009899958968162537, - "tokens_produced": 420000, - "bytes_processed": 1048000, - "avg_time_ms": 643.2205001823604, - "tokens_per_sec": 652964.2632362078, - "mb_per_sec": 1.5538228077469456, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.006600283086299896, - "tokens_produced": 378000, - "bytes_processed": 660000, - "avg_time_ms": 318.2939298450947, - "tokens_per_sec": 1187581.554520888, - "mb_per_sec": 1.9774962379409802, - "notes": "" - }, - { - "impl": "tiktoken:p50k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.004800036549568176, - "tokens_produced": 517500, - "bytes_processed": 1397500, - "avg_time_ms": 704.3563799932599, - "tokens_per_sec": 734713.2995444039, - "mb_per_sec": 1.892166941386245, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "english", - "status": "OK", - "load_time_ms": 390.31270053237677, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 278.1250501982868, - "tokens_per_sec": 503374.2911693411, - "mb_per_sec": 2.537416150172338, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.0066999346017837524, - "tokens_produced": 308000, - "bytes_processed": 1048000, - "avg_time_ms": 607.443190086633, - "tokens_per_sec": 507043.30055304983, - "mb_per_sec": 1.6453401732122626, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.00690016895532608, - "tokens_produced": 306000, - "bytes_processed": 660000, - "avg_time_ms": 358.5609101690352, - "tokens_per_sec": 853411.4883179637, - "mb_per_sec": 1.7554201558987488, - "notes": "" - }, - { - "impl": "tiktoken:cl100k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.3294004127383232, - "tokens_produced": 410000, - "bytes_processed": 1397500, - "avg_time_ms": 696.1764998733997, - "tokens_per_sec": 588931.1116858424, - "mb_per_sec": 1.9143993763364577, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "english", - "status": "OK", - "load_time_ms": 856.5244004130363, - "tokens_produced": 140001, - "bytes_processed": 740000, - "avg_time_ms": 376.90996024757624, - "tokens_per_sec": 371444.15050225594, - "mb_per_sec": 1.87238085636439, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "code", - "status": "OK", - "load_time_ms": 0.004699453711509705, - "tokens_produced": 312000, - "bytes_processed": 1048000, - "avg_time_ms": 818.470739852637, - "tokens_per_sec": 381198.721967965, - "mb_per_sec": 1.2211196258203412, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "unicode", - "status": "OK", - "load_time_ms": 0.004400499165058136, - "tokens_produced": 210000, - "bytes_processed": 660000, - "avg_time_ms": 383.4480899386108, - "tokens_per_sec": 547662.136049812, - "mb_per_sec": 1.6414869844022293, - "notes": "" - }, - { - "impl": "tiktoken:o200k_base", - "case": "mixed", - "status": "OK", - "load_time_ms": 0.004800036549568176, - "tokens_produced": 372500, - "bytes_processed": 1397500, - "avg_time_ms": 927.4989400058985, - "tokens_per_sec": 401617.7096629685, - "mb_per_sec": 1.4369394936119912, - "notes": "" - } -] \ No newline at end of file diff --git a/benchmark_results/20260316_144732/load_time_ms.png b/benchmark_results/20260316_144732/load_time_ms.png deleted file mode 100644 index 01884c0..0000000 Binary files a/benchmark_results/20260316_144732/load_time_ms.png and /dev/null differ diff --git a/benchmark_results/20260316_144732/mb_per_sec.png b/benchmark_results/20260316_144732/mb_per_sec.png deleted file mode 100644 index 5f207e5..0000000 Binary files a/benchmark_results/20260316_144732/mb_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_144732/metadata.json b/benchmark_results/20260316_144732/metadata.json deleted file mode 100644 index 93f574e..0000000 --- a/benchmark_results/20260316_144732/metadata.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "timestamp": "2026-03-16T14:47:33", - "cwd": "C:\\Users\\botma\\CRAYON", - "device_arg": "cpu", - "platform": "Windows-10-10.0.19045-SP0", - "python": "3.13.1 (tags/v3.13.1:0671451, Dec 3 2024, 19:06:28) [MSC v.1942 64 bit (AMD64)]", - "processor": "Intel64 Family 6 Model 142 Stepping 9, GenuineIntel", - "cpu_count_logical": 4, - "ram_total_bytes": 8455294976, - "nvidia_smi": null, - "rocm_smi": null, - "versions": { - "tiktoken": "0.9.0", - "transformers": "4.57.6", - "matplotlib": "3.10.7" - }, - "backends": { - "torch": "2.10.0+cpu", - "torch_cuda_is_available": false, - "torch_cuda_device_count": 0 - } -} \ No newline at end of file diff --git a/benchmark_results/20260316_144732/tokens_per_sec.png b/benchmark_results/20260316_144732/tokens_per_sec.png deleted file mode 100644 index b333b1b..0000000 Binary files a/benchmark_results/20260316_144732/tokens_per_sec.png and /dev/null differ diff --git a/benchmark_results/20260316_144732/tokens_produced.png b/benchmark_results/20260316_144732/tokens_produced.png deleted file mode 100644 index ea990ae..0000000 Binary files a/benchmark_results/20260316_144732/tokens_produced.png and /dev/null differ diff --git a/setup.py b/setup.py index daf9b90..c86b306 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def log(msg: str, level: str = "INFO") -> None: # FOR PRODUCTION/CI: Build if CUDA_HOME exists and CRAYON_FORCE_CUDA is set, # even if no GPU is found on the build machine. FORCE_CUDA = os.environ.get("CRAYON_FORCE_CUDA", "0") == "1" - TORCH_CUDA_AVAILABLE = (torch.cuda.is_available() or FORCE_CUDA) and (CUDA_HOME is not None) + TORCH_CUDA_AVAILABLE = (torch.cuda.is_available() or FORCE_CUDA) and (CUDA_HOME is not None or shutil.which("nvcc") is not None) except ImportError: TORCH_CUDA_AVAILABLE = False diff --git a/src/crayon/c_ext/__init__.py b/src/crayon/c_ext/__init__.py index 36ef38d..35f9908 100644 --- a/src/crayon/c_ext/__init__.py +++ b/src/crayon/c_ext/__init__.py @@ -71,7 +71,10 @@ def is_cuda_available() -> bool: try: from . import crayon_cuda as _cuda # Verify it's functional - _ = _cuda.get_hardware_info() + info = _cuda.get_hardware_info() + if isinstance(info, str) and "No CUDA devices found" in info: + _cuda_error = info + return False _cuda_module = _cuda return True except ImportError as e: diff --git a/src/crayon/c_ext/gpu_engine_cuda.cu b/src/crayon/c_ext/gpu_engine_cuda.cu index 2673740..db1f4fd 100644 --- a/src/crayon/c_ext/gpu_engine_cuda.cu +++ b/src/crayon/c_ext/gpu_engine_cuda.cu @@ -53,7 +53,6 @@ __global__ void tokenize_kernel( int end = offsets[idx + 1]; int len = end - start; - int node = 0; int count = 0; int write_pos = idx * max_tokens; int pos = 0; diff --git a/src/crayon/core/vocabulary.py b/src/crayon/core/vocabulary.py index 75f2747..fa370c6 100644 --- a/src/crayon/core/vocabulary.py +++ b/src/crayon/core/vocabulary.py @@ -158,6 +158,8 @@ def _detect_cuda_availability() -> Tuple[bool, Optional[str]]: try: from ..c_ext import crayon_cuda info = crayon_cuda.get_hardware_info() + if isinstance(info, str) and "No CUDA devices found" in info: + return False, info if isinstance(info, dict) and info.get("name"): return True, None return True, None @@ -508,6 +510,10 @@ def _init_selected_backend(self) -> None: try: from ..c_ext import crayon_cuda info = crayon_cuda.get_hardware_info() + + if isinstance(info, str) and "No CUDA devices found" in info: + raise RuntimeError(info) + self._gpu_backend = crayon_cuda self._device_state = DeviceState.READY