diff --git a/benchmark_results/20260317_053035/benchmark_results.csv b/benchmark_results/20260317_053035/benchmark_results.csv new file mode 100644 index 0000000..98156d7 --- /dev/null +++ b/benchmark_results/20260317_053035/benchmark_results.csv @@ -0,0 +1,81 @@ +impl,case,status,cold_load_time_ms,warm_load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes +crayon:cpu:lite,english,OK,14.868305000049986,9.87663299997621,144001,740000,8.647791299995333,16651766.330216331,81.60684846094816, +crayon:cpu:lite,code,OK,9.026592999930472,8.920501999909902,556000,1048000,19.12290340000027,29075082.814045496,52.26458883820623, +crayon:cpu:lite,unicode,OK,12.342592999971203,7.844900999998572,474000,660000,12.112038599980224,39134617.68531467,51.966895880694494, +crayon:cpu:lite,mixed,OK,12.142413999981727,8.46946800004389,640000,1397500,26.57187120001936,24085620.285542168,50.156793518431755, +crayon:cpu:standard,english,OK,56.67616999994607,56.14077899997483,136001,740000,8.373979399993914,16240904.533404853,84.2752245295872, +crayon:cpu:standard,code,OK,59.37171699997634,60.828852999975425,312000,1048000,19.83747910001057,15727804.849953631,50.38194009203607, +crayon:cpu:standard,unicode,OK,55.76143799999045,60.44815200004905,216001,660000,8.230683800013594,26243384.541104987,76.47299594076077, +crayon:cpu:standard,mixed,OK,53.84370900003432,59.3777520000458,372501,1397500,24.60571969999137,15138797.179752098,54.16463624830295, +crayon:cpu:lite,english,OK,12.34249699996326,8.38803600004212,144001,740000,8.31000389998735,17328632.059994485,84.92402682767685, +crayon:cpu:lite,code,OK,8.134915000027831,8.08777400004601,556000,1048000,18.63192479999043,29841253.97502064,53.64183756228253, +crayon:cpu:lite,unicode,OK,8.606406000012612,8.018338999931984,474000,660000,11.482100199975775,41281646.36648964,54.81793729943699, +crayon:cpu:lite,mixed,OK,8.287248999977237,8.105816000011146,640000,1397500,25.149739800031057,25447579.382082105,52.99298791059813, +crayon:cpu:standard,english,OK,57.03588000005766,55.99314200003391,136001,740000,8.245439300003454,16494087.8286428,85.5890109021031, +crayon:cpu:standard,code,OK,52.5738100000126,55.370928000002095,312000,1048000,19.90267890000723,15676281.648692358,50.216892339723515, +crayon:cpu:standard,unicode,OK,51.06038899998566,56.10423499990702,216001,660000,8.22087299998202,26274703.42875658,76.56425890893844, +crayon:cpu:standard,mixed,OK,50.55269899992254,56.21722000000773,372501,1397500,24.131329999977424,15436405.701647962,55.229440614296074, +crayon:cpu:lite,english,OK,12.330100999975002,8.000930999969569,144001,740000,10.23242560000881,14073007.283813138,68.968885944308, +crayon:cpu:lite,code,OK,8.94056599997839,7.907703000000765,556000,1048000,40.71248950000381,13656742.852827704,24.548994568206307, +crayon:cpu:lite,unicode,OK,11.877349999963371,8.837169999992511,474000,660000,13.601785399976052,34848366.303502664,46.27517861215728, +crayon:cpu:lite,mixed,OK,10.845587999938289,9.425191999980598,640000,1397500,29.756448500006627,21507943.05979954,44.788942375883245, +crayon:cpu:standard,english,OK,68.05869800007258,71.99046100004125,136001,740000,9.61803189999273,14140210.951068152,73.37457407904402, +crayon:cpu:standard,code,OK,61.89855200000238,63.58751300001586,312000,1048000,22.32071649999625,13978045.91084934,44.7768190413564, +crayon:cpu:standard,unicode,OK,63.469779000001836,61.042190000080154,216001,660000,10.250069699998221,21073124.995436616,61.406904269951866, +crayon:cpu:standard,mixed,OK,78.44032599996353,67.1652720000111,372501,1397500,29.630535900025734,12571524.229491796,44.979269415663076, +crayon:cpu:lite,english,OK,14.983427999936794,9.01762300009068,144001,740000,8.486741399985931,16967761.030192193,83.15547285814493, +crayon:cpu:lite,code,OK,10.302792999937083,12.465697000038745,556000,1048000,21.7420526000069,25572562.54636343,45.968552370875635, +crayon:cpu:lite,unicode,OK,8.5818789999621,10.996005000038167,474000,660000,14.206197699979839,33365718.963679682,44.30636980569532, +crayon:cpu:lite,mixed,OK,8.1939039999952,7.966666000015721,640000,1397500,31.072842300000048,20596764.010867424,42.8914691585112, +crayon:cpu:standard,english,OK,75.51630000000387,69.93360599994958,136001,740000,10.2323704000014,13291250.676381044,68.96925800697446, +crayon:cpu:standard,code,OK,68.99347599994599,75.94491300005757,312000,1048000,23.726231499995265,13150002.350776281,42.124291149816585, +crayon:cpu:standard,unicode,OK,62.730942000030154,73.37964800001373,216001,660000,11.02782569998908,19586907.326637734,57.07607881658379, +crayon:cpu:standard,mixed,OK,62.92240700008733,67.83681499996419,372501,1397500,25.266263900005015,14743018.65421132,52.74859244929638, +crayon:cpu:lite,english,OK,14.862322999988464,8.51527999998325,144001,740000,9.711931699996512,14827225.360331943,72.66515209748421, +crayon:cpu:lite,code,OK,12.431329000037294,9.098590999997214,556000,1048000,20.486216000006152,27140200.02521857,48.78649544617951, +crayon:cpu:lite,unicode,OK,10.318500999915159,9.489746999975068,474000,660000,13.82918060000975,34275349.61830391,45.51426921329534, +crayon:cpu:lite,mixed,OK,8.992912999929104,8.828263999930641,640000,1397500,30.61809850001964,20902669.706924792,43.528498583178816, +crayon:cpu:standard,english,OK,63.77301999998508,68.11127399998895,136001,740000,9.099823600001855,14945454.54704993,77.55304115350995, +crayon:cpu:standard,code,OK,70.96236099994258,65.23993500002234,312000,1048000,21.603614000014204,14442028.079181328,46.26312447505741, +crayon:cpu:standard,unicode,OK,64.17608200001723,68.946622999988,216001,660000,9.849700700010544,21929701.884217534,63.902961927305185, +crayon:cpu:standard,mixed,OK,61.973479000016596,66.58864000007725,372501,1397500,26.984551500004272,13804231.654542822,49.38973535200403, +crayon:cpu:lite,english,OK,14.038064000033046,8.293565999906605,144001,740000,9.331666100001712,15431435.1217489,75.62625865283538, +crayon:cpu:lite,code,OK,9.214194000037423,11.521090999963235,556000,1048000,20.19936870000265,27525612.718774077,49.47930296423664, +crayon:cpu:lite,unicode,OK,10.631344999978865,9.76838700000826,474000,660000,14.771757600021829,32088260.099752758,42.61003097066762, +crayon:cpu:lite,mixed,OK,11.537464999946678,10.287465999908818,640000,1397500,30.200901099999555,21191420.675855577,44.129804364604006, +crayon:cpu:standard,english,OK,70.90438300008373,68.66426600004161,136001,740000,8.855051299974548,15358578.442158874,79.69677082984866, +crayon:cpu:standard,code,OK,67.25170300001082,65.2393130000064,312000,1048000,20.73662330002435,15045844.035737181,48.197368931931, +crayon:cpu:standard,unicode,OK,67.00485500005016,68.079785000009,216001,660000,8.824563100017713,24477245.791303415,71.32648287447358, +crayon:cpu:standard,mixed,OK,58.914462999950956,68.05693100000099,372501,1397500,26.207790499984185,14213369.112524949,50.853575664020156, +crayon:cpu:lite,english,OK,12.835383000037837,9.33742900008383,144001,740000,8.70022369997514,16551413.499909371,81.11504008139946, +crayon:cpu:lite,code,OK,9.479603000045245,8.138844999962203,556000,1048000,21.23308629999201,26185547.976612765,47.07043853507656, +crayon:cpu:lite,unicode,OK,8.614314000055856,9.598793999998634,474000,660000,15.694109399998979,30202414.671585687,40.10581504090738, +crayon:cpu:lite,mixed,OK,11.087617000043792,8.254261000047336,640000,1397500,29.928679299996475,21384171.135145124,44.53119510615663, +crayon:cpu:standard,english,OK,74.17762199997924,73.78222199997708,136001,740000,8.760950500015952,15523543.935073296,80.55278866594898, +crayon:cpu:standard,code,OK,66.53267000001506,75.45856199999434,312000,1048000,20.645504200001596,15112248.99026568,48.41008841012818, +crayon:cpu:standard,unicode,OK,65.45338000000811,88.04373200007376,216001,660000,8.822801700000582,24482132.472725276,71.34072262194032, +crayon:cpu:standard,mixed,OK,72.09946499995112,70.31428500010861,372501,1397500,27.636978300017745,13478354.831568573,48.22379070207102, +crayon:cpu:lite,english,OK,21.455864000017755,9.034896000002846,144001,740000,10.249723800006905,14049256.624837343,68.85248889732517, +crayon:cpu:lite,code,OK,11.17282100005923,11.382386999912342,556000,1048000,21.1772855999925,26254545.105638888,47.19446592315419, +crayon:cpu:lite,unicode,OK,8.655612999973528,10.659256999929312,474000,660000,15.423255300004257,30732811.639308672,40.81012967657686, +crayon:cpu:lite,mixed,OK,10.22083299994847,9.119001000044591,640000,1397500,28.74799730000177,22262420.3460587,46.360094001318565, +crayon:cpu:standard,english,OK,58.88794100007999,56.890226000064104,136001,740000,7.4119927999959145,18348776.59353298,95.21312461893028, +crayon:cpu:standard,code,OK,54.90327600000455,55.97901500004809,312000,1048000,17.434602800005905,17895446.4050018,57.32569276504604, +crayon:cpu:standard,unicode,OK,50.7556890000842,55.61820000002626,216001,660000,7.618974400008938,28350403.697346274,82.61283156790586, +crayon:cpu:standard,mixed,OK,50.34348299989233,55.09503400003268,372501,1397500,21.914746300012666,16997732.709312025,60.815664435821645, +crayon:cpu:lite,english,OK,12.457188999974278,8.017877000042972,144001,740000,7.411144900015643,19430331.202901736,95.22401783550818, +crayon:cpu:lite,code,OK,8.076985000002423,7.703710999976465,556000,1048000,16.831052300005922,33034179.330534454,59.381354521333066, +crayon:cpu:lite,unicode,OK,8.161683000025732,7.643239999993057,474000,660000,11.583916900008262,40918801.825974934,54.33611569051191, +crayon:cpu:lite,mixed,OK,8.703353000100833,8.032742999944276,640000,1397500,24.614256099994236,26001192.049031693,54.14585156518488, +crayon:cpu:standard,english,OK,55.18846500001473,55.58668399999078,136001,740000,7.250363000014204,18757819.436038382,97.33567741908128, +crayon:cpu:standard,code,OK,52.35724600004232,54.53782400002183,312000,1048000,17.24318060001906,18094109.621496115,57.9620840712325, +crayon:cpu:standard,unicode,OK,50.46295099998588,55.4346459999806,216001,660000,7.501434900018467,28794624.34573261,83.90728670145168, +crayon:cpu:standard,mixed,OK,49.663459000043986,54.793023000001995,372501,1397500,21.751640299999053,17125191.243623875,61.27169440080307, +crayon:cpu:lite,english,OK,12.217731999953685,8.143410000002405,144001,740000,7.4156978999667444,19418401.604607675,95.16555335186858, +crayon:cpu:lite,code,OK,8.170772999960718,7.94540800006871,556000,1048000,16.79275569999845,33109515.194105472,59.516776248572604, +crayon:cpu:lite,unicode,OK,8.131237000043257,7.8823059999422185,474000,660000,11.37300549999054,41677637.45479541,55.343774240560116, +crayon:cpu:lite,mixed,OK,8.195090999947752,7.952807000037865,640000,1397500,23.71097740000323,26991717.346915983,56.20855836915238, +crayon:cpu:standard,english,OK,55.76155500000368,55.5743900000607,136001,740000,7.294867399991745,18643382.057932116,96.74185361360011, +crayon:cpu:standard,code,OK,52.74450500007788,54.82426399998985,312000,1048000,17.348223099997995,17984550.82123287,57.61112696284529, +crayon:cpu:standard,unicode,OK,50.57152599999881,55.03631899989614,216001,660000,7.481271000028755,28872233.073654167,84.13343786446256, +crayon:cpu:standard,mixed,OK,49.62355600002866,55.199267000034524,372501,1397500,21.733554900015406,17139441.831475805,61.3226811402487, diff --git a/benchmark_results/20260317_053035/benchmark_results.json b/benchmark_results/20260317_053035/benchmark_results.json new file mode 100644 index 0000000..f521e30 --- /dev/null +++ b/benchmark_results/20260317_053035/benchmark_results.json @@ -0,0 +1,1042 @@ +[ + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 14.868305000049986, + "warm_load_time_ms": 9.87663299997621, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 8.647791299995333, + "tokens_per_sec": 16651766.330216331, + "mb_per_sec": 81.60684846094816, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 9.026592999930472, + "warm_load_time_ms": 8.920501999909902, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 19.12290340000027, + "tokens_per_sec": 29075082.814045496, + "mb_per_sec": 52.26458883820623, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 12.342592999971203, + "warm_load_time_ms": 7.844900999998572, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 12.112038599980224, + "tokens_per_sec": 39134617.68531467, + "mb_per_sec": 51.966895880694494, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 12.142413999981727, + "warm_load_time_ms": 8.46946800004389, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 26.57187120001936, + "tokens_per_sec": 24085620.285542168, + "mb_per_sec": 50.156793518431755, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 56.67616999994607, + "warm_load_time_ms": 56.14077899997483, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 8.373979399993914, + "tokens_per_sec": 16240904.533404853, + "mb_per_sec": 84.2752245295872, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 59.37171699997634, + "warm_load_time_ms": 60.828852999975425, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 19.83747910001057, + "tokens_per_sec": 15727804.849953631, + "mb_per_sec": 50.38194009203607, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 55.76143799999045, + "warm_load_time_ms": 60.44815200004905, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 8.230683800013594, + "tokens_per_sec": 26243384.541104987, + "mb_per_sec": 76.47299594076077, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 53.84370900003432, + "warm_load_time_ms": 59.3777520000458, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 24.60571969999137, + "tokens_per_sec": 15138797.179752098, + "mb_per_sec": 54.16463624830295, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.34249699996326, + "warm_load_time_ms": 8.38803600004212, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 8.31000389998735, + "tokens_per_sec": 17328632.059994485, + "mb_per_sec": 84.92402682767685, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.134915000027831, + "warm_load_time_ms": 8.08777400004601, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 18.63192479999043, + "tokens_per_sec": 29841253.97502064, + "mb_per_sec": 53.64183756228253, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.606406000012612, + "warm_load_time_ms": 8.018338999931984, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.482100199975775, + "tokens_per_sec": 41281646.36648964, + "mb_per_sec": 54.81793729943699, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.287248999977237, + "warm_load_time_ms": 8.105816000011146, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 25.149739800031057, + "tokens_per_sec": 25447579.382082105, + "mb_per_sec": 52.99298791059813, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 57.03588000005766, + "warm_load_time_ms": 55.99314200003391, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 8.245439300003454, + "tokens_per_sec": 16494087.8286428, + "mb_per_sec": 85.5890109021031, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 52.5738100000126, + "warm_load_time_ms": 55.370928000002095, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 19.90267890000723, + "tokens_per_sec": 15676281.648692358, + "mb_per_sec": 50.216892339723515, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 51.06038899998566, + "warm_load_time_ms": 56.10423499990702, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 8.22087299998202, + "tokens_per_sec": 26274703.42875658, + "mb_per_sec": 76.56425890893844, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 50.55269899992254, + "warm_load_time_ms": 56.21722000000773, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 24.131329999977424, + "tokens_per_sec": 15436405.701647962, + "mb_per_sec": 55.229440614296074, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.330100999975002, + "warm_load_time_ms": 8.000930999969569, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 10.23242560000881, + "tokens_per_sec": 14073007.283813138, + "mb_per_sec": 68.968885944308, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.94056599997839, + "warm_load_time_ms": 7.907703000000765, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 40.71248950000381, + "tokens_per_sec": 13656742.852827704, + "mb_per_sec": 24.548994568206307, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 11.877349999963371, + "warm_load_time_ms": 8.837169999992511, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 13.601785399976052, + "tokens_per_sec": 34848366.303502664, + "mb_per_sec": 46.27517861215728, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 10.845587999938289, + "warm_load_time_ms": 9.425191999980598, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 29.756448500006627, + "tokens_per_sec": 21507943.05979954, + "mb_per_sec": 44.788942375883245, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 68.05869800007258, + "warm_load_time_ms": 71.99046100004125, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 9.61803189999273, + "tokens_per_sec": 14140210.951068152, + "mb_per_sec": 73.37457407904402, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 61.89855200000238, + "warm_load_time_ms": 63.58751300001586, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 22.32071649999625, + "tokens_per_sec": 13978045.91084934, + "mb_per_sec": 44.7768190413564, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 63.469779000001836, + "warm_load_time_ms": 61.042190000080154, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 10.250069699998221, + "tokens_per_sec": 21073124.995436616, + "mb_per_sec": 61.406904269951866, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 78.44032599996353, + "warm_load_time_ms": 67.1652720000111, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 29.630535900025734, + "tokens_per_sec": 12571524.229491796, + "mb_per_sec": 44.979269415663076, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 14.983427999936794, + "warm_load_time_ms": 9.01762300009068, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 8.486741399985931, + "tokens_per_sec": 16967761.030192193, + "mb_per_sec": 83.15547285814493, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 10.302792999937083, + "warm_load_time_ms": 12.465697000038745, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 21.7420526000069, + "tokens_per_sec": 25572562.54636343, + "mb_per_sec": 45.968552370875635, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.5818789999621, + "warm_load_time_ms": 10.996005000038167, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 14.206197699979839, + "tokens_per_sec": 33365718.963679682, + "mb_per_sec": 44.30636980569532, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.1939039999952, + "warm_load_time_ms": 7.966666000015721, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 31.072842300000048, + "tokens_per_sec": 20596764.010867424, + "mb_per_sec": 42.8914691585112, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 75.51630000000387, + "warm_load_time_ms": 69.93360599994958, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 10.2323704000014, + "tokens_per_sec": 13291250.676381044, + "mb_per_sec": 68.96925800697446, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 68.99347599994599, + "warm_load_time_ms": 75.94491300005757, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 23.726231499995265, + "tokens_per_sec": 13150002.350776281, + "mb_per_sec": 42.124291149816585, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 62.730942000030154, + "warm_load_time_ms": 73.37964800001373, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 11.02782569998908, + "tokens_per_sec": 19586907.326637734, + "mb_per_sec": 57.07607881658379, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 62.92240700008733, + "warm_load_time_ms": 67.83681499996419, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 25.266263900005015, + "tokens_per_sec": 14743018.65421132, + "mb_per_sec": 52.74859244929638, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 14.862322999988464, + "warm_load_time_ms": 8.51527999998325, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 9.711931699996512, + "tokens_per_sec": 14827225.360331943, + "mb_per_sec": 72.66515209748421, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 12.431329000037294, + "warm_load_time_ms": 9.098590999997214, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 20.486216000006152, + "tokens_per_sec": 27140200.02521857, + "mb_per_sec": 48.78649544617951, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 10.318500999915159, + "warm_load_time_ms": 9.489746999975068, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 13.82918060000975, + "tokens_per_sec": 34275349.61830391, + "mb_per_sec": 45.51426921329534, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.992912999929104, + "warm_load_time_ms": 8.828263999930641, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 30.61809850001964, + "tokens_per_sec": 20902669.706924792, + "mb_per_sec": 43.528498583178816, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 63.77301999998508, + "warm_load_time_ms": 68.11127399998895, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 9.099823600001855, + "tokens_per_sec": 14945454.54704993, + "mb_per_sec": 77.55304115350995, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 70.96236099994258, + "warm_load_time_ms": 65.23993500002234, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 21.603614000014204, + "tokens_per_sec": 14442028.079181328, + "mb_per_sec": 46.26312447505741, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 64.17608200001723, + "warm_load_time_ms": 68.946622999988, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 9.849700700010544, + "tokens_per_sec": 21929701.884217534, + "mb_per_sec": 63.902961927305185, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 61.973479000016596, + "warm_load_time_ms": 66.58864000007725, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 26.984551500004272, + "tokens_per_sec": 13804231.654542822, + "mb_per_sec": 49.38973535200403, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 14.038064000033046, + "warm_load_time_ms": 8.293565999906605, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 9.331666100001712, + "tokens_per_sec": 15431435.1217489, + "mb_per_sec": 75.62625865283538, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 9.214194000037423, + "warm_load_time_ms": 11.521090999963235, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 20.19936870000265, + "tokens_per_sec": 27525612.718774077, + "mb_per_sec": 49.47930296423664, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 10.631344999978865, + "warm_load_time_ms": 9.76838700000826, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 14.771757600021829, + "tokens_per_sec": 32088260.099752758, + "mb_per_sec": 42.61003097066762, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 11.537464999946678, + "warm_load_time_ms": 10.287465999908818, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 30.200901099999555, + "tokens_per_sec": 21191420.675855577, + "mb_per_sec": 44.129804364604006, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 70.90438300008373, + "warm_load_time_ms": 68.66426600004161, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 8.855051299974548, + "tokens_per_sec": 15358578.442158874, + "mb_per_sec": 79.69677082984866, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 67.25170300001082, + "warm_load_time_ms": 65.2393130000064, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 20.73662330002435, + "tokens_per_sec": 15045844.035737181, + "mb_per_sec": 48.197368931931, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 67.00485500005016, + "warm_load_time_ms": 68.079785000009, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 8.824563100017713, + "tokens_per_sec": 24477245.791303415, + "mb_per_sec": 71.32648287447358, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 58.914462999950956, + "warm_load_time_ms": 68.05693100000099, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 26.207790499984185, + "tokens_per_sec": 14213369.112524949, + "mb_per_sec": 50.853575664020156, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.835383000037837, + "warm_load_time_ms": 9.33742900008383, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 8.70022369997514, + "tokens_per_sec": 16551413.499909371, + "mb_per_sec": 81.11504008139946, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 9.479603000045245, + "warm_load_time_ms": 8.138844999962203, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 21.23308629999201, + "tokens_per_sec": 26185547.976612765, + "mb_per_sec": 47.07043853507656, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.614314000055856, + "warm_load_time_ms": 9.598793999998634, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 15.694109399998979, + "tokens_per_sec": 30202414.671585687, + "mb_per_sec": 40.10581504090738, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 11.087617000043792, + "warm_load_time_ms": 8.254261000047336, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 29.928679299996475, + "tokens_per_sec": 21384171.135145124, + "mb_per_sec": 44.53119510615663, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 74.17762199997924, + "warm_load_time_ms": 73.78222199997708, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 8.760950500015952, + "tokens_per_sec": 15523543.935073296, + "mb_per_sec": 80.55278866594898, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 66.53267000001506, + "warm_load_time_ms": 75.45856199999434, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 20.645504200001596, + "tokens_per_sec": 15112248.99026568, + "mb_per_sec": 48.41008841012818, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 65.45338000000811, + "warm_load_time_ms": 88.04373200007376, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 8.822801700000582, + "tokens_per_sec": 24482132.472725276, + "mb_per_sec": 71.34072262194032, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 72.09946499995112, + "warm_load_time_ms": 70.31428500010861, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 27.636978300017745, + "tokens_per_sec": 13478354.831568573, + "mb_per_sec": 48.22379070207102, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 21.455864000017755, + "warm_load_time_ms": 9.034896000002846, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 10.249723800006905, + "tokens_per_sec": 14049256.624837343, + "mb_per_sec": 68.85248889732517, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 11.17282100005923, + "warm_load_time_ms": 11.382386999912342, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 21.1772855999925, + "tokens_per_sec": 26254545.105638888, + "mb_per_sec": 47.19446592315419, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.655612999973528, + "warm_load_time_ms": 10.659256999929312, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 15.423255300004257, + "tokens_per_sec": 30732811.639308672, + "mb_per_sec": 40.81012967657686, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 10.22083299994847, + "warm_load_time_ms": 9.119001000044591, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 28.74799730000177, + "tokens_per_sec": 22262420.3460587, + "mb_per_sec": 46.360094001318565, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 58.88794100007999, + "warm_load_time_ms": 56.890226000064104, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.4119927999959145, + "tokens_per_sec": 18348776.59353298, + "mb_per_sec": 95.21312461893028, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 54.90327600000455, + "warm_load_time_ms": 55.97901500004809, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 17.434602800005905, + "tokens_per_sec": 17895446.4050018, + "mb_per_sec": 57.32569276504604, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 50.7556890000842, + "warm_load_time_ms": 55.61820000002626, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.618974400008938, + "tokens_per_sec": 28350403.697346274, + "mb_per_sec": 82.61283156790586, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 50.34348299989233, + "warm_load_time_ms": 55.09503400003268, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 21.914746300012666, + "tokens_per_sec": 16997732.709312025, + "mb_per_sec": 60.815664435821645, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.457188999974278, + "warm_load_time_ms": 8.017877000042972, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.411144900015643, + "tokens_per_sec": 19430331.202901736, + "mb_per_sec": 95.22401783550818, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.076985000002423, + "warm_load_time_ms": 7.703710999976465, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 16.831052300005922, + "tokens_per_sec": 33034179.330534454, + "mb_per_sec": 59.381354521333066, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.161683000025732, + "warm_load_time_ms": 7.643239999993057, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.583916900008262, + "tokens_per_sec": 40918801.825974934, + "mb_per_sec": 54.33611569051191, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.703353000100833, + "warm_load_time_ms": 8.032742999944276, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 24.614256099994236, + "tokens_per_sec": 26001192.049031693, + "mb_per_sec": 54.14585156518488, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 55.18846500001473, + "warm_load_time_ms": 55.58668399999078, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.250363000014204, + "tokens_per_sec": 18757819.436038382, + "mb_per_sec": 97.33567741908128, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 52.35724600004232, + "warm_load_time_ms": 54.53782400002183, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 17.24318060001906, + "tokens_per_sec": 18094109.621496115, + "mb_per_sec": 57.9620840712325, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 50.46295099998588, + "warm_load_time_ms": 55.4346459999806, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.501434900018467, + "tokens_per_sec": 28794624.34573261, + "mb_per_sec": 83.90728670145168, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 49.663459000043986, + "warm_load_time_ms": 54.793023000001995, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 21.751640299999053, + "tokens_per_sec": 17125191.243623875, + "mb_per_sec": 61.27169440080307, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.217731999953685, + "warm_load_time_ms": 8.143410000002405, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.4156978999667444, + "tokens_per_sec": 19418401.604607675, + "mb_per_sec": 95.16555335186858, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.170772999960718, + "warm_load_time_ms": 7.94540800006871, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 16.79275569999845, + "tokens_per_sec": 33109515.194105472, + "mb_per_sec": 59.516776248572604, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.131237000043257, + "warm_load_time_ms": 7.8823059999422185, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.37300549999054, + "tokens_per_sec": 41677637.45479541, + "mb_per_sec": 55.343774240560116, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.195090999947752, + "warm_load_time_ms": 7.952807000037865, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 23.71097740000323, + "tokens_per_sec": 26991717.346915983, + "mb_per_sec": 56.20855836915238, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 55.76155500000368, + "warm_load_time_ms": 55.5743900000607, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.294867399991745, + "tokens_per_sec": 18643382.057932116, + "mb_per_sec": 96.74185361360011, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 52.74450500007788, + "warm_load_time_ms": 54.82426399998985, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 17.348223099997995, + "tokens_per_sec": 17984550.82123287, + "mb_per_sec": 57.61112696284529, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 50.57152599999881, + "warm_load_time_ms": 55.03631899989614, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.481271000028755, + "tokens_per_sec": 28872233.073654167, + "mb_per_sec": 84.13343786446256, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 49.62355600002866, + "warm_load_time_ms": 55.199267000034524, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 21.733554900015406, + "tokens_per_sec": 17139441.831475805, + "mb_per_sec": 61.3226811402487, + "notes": "" + } +] \ No newline at end of file diff --git a/benchmark_results/20260317_053035/benchmark_summary.csv b/benchmark_results/20260317_053035/benchmark_summary.csv new file mode 100644 index 0000000..e7e34c4 --- /dev/null +++ b/benchmark_results/20260317_053035/benchmark_summary.csv @@ -0,0 +1,9 @@ +impl,case,n,tokens_per_sec_mean,tokens_per_sec_std,cold_load_time_ms_mean,cold_load_time_ms_std,warm_load_time_ms_mean,warm_load_time_ms_std,mb_per_sec_mean,mb_per_sec_std,tokens_produced_mean,tokens_produced_std +crayon:cpu:lite,code,10,27139524.253914148,5451977.904253478,9.495057200001611,1.4263281122069225,9.317170899987559,1.7830116864459251,48.785280697812325,9.80032921464777,556000.0,0.0 +crayon:cpu:lite,english,10,16472923.01185531,1939763.093216966,14.23908859999301,2.787539039221978,8.662568100010049,0.6280678015854914,80.73037450074989,9.506376060000866,144001.0,0.0 +crayon:cpu:lite,mixed,10,23037149.799822308,2378674.551117097,9.820642699980908,1.5175805267599374,8.644168399996488,0.7712172541295034,47.97341949530196,4.953440554717397,640000.0,0.0 +crayon:cpu:lite,unicode,10,35852562.4628708,4494830.489626035,9.592092099990168,1.5776457100921666,9.073814599980778,1.2149975749237105,47.60865164305033,5.968689663305935,474000.0,0.0 +crayon:cpu:standard,code,10,15710636.27131866,1751018.7873777037,60.75893160000305,7.354040464034685,62.70111200001338,8.047995840275314,50.3269428239173,5.609156807788926,312000.0,0.0 +crayon:cpu:standard,english,10,16174400.900128242,1904801.6981369553,63.59800340002266,7.976172307598718,63.26670500001228,7.7905110097749555,83.9301323818628,9.884153340391368,136001.0,0.0 +crayon:cpu:standard,mixed,10,15064806.714815121,1618637.5865232626,58.83770459998914,10.118526728955352,62.06442390002849,6.4443090864788815,53.899908042252704,5.7912735768152475,372501.0,0.0 +crayon:cpu:standard,unicode,10,25008446.15569152,3307883.466645642,58.14470310001525,7.027832893673329,64.21335300000237,10.616612670839388,72.8743961493774,9.639143858182047,216001.0,0.0 diff --git a/benchmark_results/20260317_053035/benchmark_summary.json b/benchmark_results/20260317_053035/benchmark_summary.json new file mode 100644 index 0000000..06ef0cb --- /dev/null +++ b/benchmark_results/20260317_053035/benchmark_summary.json @@ -0,0 +1,122 @@ +[ + { + "impl": "crayon:cpu:lite", + "case": "code", + "n": 10, + "tokens_per_sec_mean": 27139524.253914148, + "tokens_per_sec_std": 5451977.904253478, + "cold_load_time_ms_mean": 9.495057200001611, + "cold_load_time_ms_std": 1.4263281122069225, + "warm_load_time_ms_mean": 9.317170899987559, + "warm_load_time_ms_std": 1.7830116864459251, + "mb_per_sec_mean": 48.785280697812325, + "mb_per_sec_std": 9.80032921464777, + "tokens_produced_mean": 556000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "n": 10, + "tokens_per_sec_mean": 16472923.01185531, + "tokens_per_sec_std": 1939763.093216966, + "cold_load_time_ms_mean": 14.23908859999301, + "cold_load_time_ms_std": 2.787539039221978, + "warm_load_time_ms_mean": 8.662568100010049, + "warm_load_time_ms_std": 0.6280678015854914, + "mb_per_sec_mean": 80.73037450074989, + "mb_per_sec_std": 9.506376060000866, + "tokens_produced_mean": 144001.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "n": 10, + "tokens_per_sec_mean": 23037149.799822308, + "tokens_per_sec_std": 2378674.551117097, + "cold_load_time_ms_mean": 9.820642699980908, + "cold_load_time_ms_std": 1.5175805267599374, + "warm_load_time_ms_mean": 8.644168399996488, + "warm_load_time_ms_std": 0.7712172541295034, + "mb_per_sec_mean": 47.97341949530196, + "mb_per_sec_std": 4.953440554717397, + "tokens_produced_mean": 640000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "n": 10, + "tokens_per_sec_mean": 35852562.4628708, + "tokens_per_sec_std": 4494830.489626035, + "cold_load_time_ms_mean": 9.592092099990168, + "cold_load_time_ms_std": 1.5776457100921666, + "warm_load_time_ms_mean": 9.073814599980778, + "warm_load_time_ms_std": 1.2149975749237105, + "mb_per_sec_mean": 47.60865164305033, + "mb_per_sec_std": 5.968689663305935, + "tokens_produced_mean": 474000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "n": 10, + "tokens_per_sec_mean": 15710636.27131866, + "tokens_per_sec_std": 1751018.7873777037, + "cold_load_time_ms_mean": 60.75893160000305, + "cold_load_time_ms_std": 7.354040464034685, + "warm_load_time_ms_mean": 62.70111200001338, + "warm_load_time_ms_std": 8.047995840275314, + "mb_per_sec_mean": 50.3269428239173, + "mb_per_sec_std": 5.609156807788926, + "tokens_produced_mean": 312000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "n": 10, + "tokens_per_sec_mean": 16174400.900128242, + "tokens_per_sec_std": 1904801.6981369553, + "cold_load_time_ms_mean": 63.59800340002266, + "cold_load_time_ms_std": 7.976172307598718, + "warm_load_time_ms_mean": 63.26670500001228, + "warm_load_time_ms_std": 7.7905110097749555, + "mb_per_sec_mean": 83.9301323818628, + "mb_per_sec_std": 9.884153340391368, + "tokens_produced_mean": 136001.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "n": 10, + "tokens_per_sec_mean": 15064806.714815121, + "tokens_per_sec_std": 1618637.5865232626, + "cold_load_time_ms_mean": 58.83770459998914, + "cold_load_time_ms_std": 10.118526728955352, + "warm_load_time_ms_mean": 62.06442390002849, + "warm_load_time_ms_std": 6.4443090864788815, + "mb_per_sec_mean": 53.899908042252704, + "mb_per_sec_std": 5.7912735768152475, + "tokens_produced_mean": 372501.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "n": 10, + "tokens_per_sec_mean": 25008446.15569152, + "tokens_per_sec_std": 3307883.466645642, + "cold_load_time_ms_mean": 58.14470310001525, + "cold_load_time_ms_std": 7.027832893673329, + "warm_load_time_ms_mean": 64.21335300000237, + "warm_load_time_ms_std": 10.616612670839388, + "mb_per_sec_mean": 72.8743961493774, + "mb_per_sec_std": 9.639143858182047, + "tokens_produced_mean": 216001.0, + "tokens_produced_std": 0.0 + } +] \ No newline at end of file diff --git a/benchmark_results/20260317_053035/metadata.json b/benchmark_results/20260317_053035/metadata.json new file mode 100644 index 0000000..931c263 --- /dev/null +++ b/benchmark_results/20260317_053035/metadata.json @@ -0,0 +1,22 @@ +{ + "timestamp": "2026-03-17T05:30:35", + "cwd": "/app", + "device_arg": "cpu", + "platform": "Linux-6.8.0-x86_64-with-glibc2.39", + "python": "3.12.13 (main, Mar 6 2026, 16:37:31) [GCC 13.3.0]", + "processor": "x86_64", + "cpu_count_logical": 4, + "ram_total_bytes": null, + "nvidia_smi": null, + "rocm_smi": null, + "versions": { + "tiktoken": null, + "transformers": null, + "matplotlib": null + }, + "backends": { + "torch": "2.10.0+cu128", + "torch_cuda_is_available": false, + "torch_cuda_device_count": 0 + } +} \ No newline at end of file diff --git a/benchmark_results/20260317_053838/benchmark_results.csv b/benchmark_results/20260317_053838/benchmark_results.csv new file mode 100644 index 0000000..6728d05 --- /dev/null +++ b/benchmark_results/20260317_053838/benchmark_results.csv @@ -0,0 +1,81 @@ +impl,case,status,cold_load_time_ms,warm_load_time_ms,tokens_produced,bytes_processed,avg_time_ms,tokens_per_sec,mb_per_sec,notes +crayon:cpu:lite,english,OK,13.410531000090486,9.785870999849067,144001,740000,8.480945400037854,16979357.041887958,83.21230250314724, +crayon:cpu:lite,code,OK,8.735390000083498,8.888432000048851,556000,1048000,19.00110249998761,29261459.960039824,52.599615395706735, +crayon:cpu:lite,unicode,OK,12.106269000014436,8.023800000046322,474000,660000,11.997211600009905,39509180.61656999,52.464278351780116, +crayon:cpu:lite,mixed,OK,12.21738499998537,8.244456000056744,640000,1397500,27.57976560003499,23205418.395549674,48.32382829156617, +crayon:cpu:standard,english,OK,58.660766000002695,56.090405000077226,136001,740000,8.652120299984745,15718805.944045855,81.56601730813536, +crayon:cpu:standard,code,OK,56.99175500012643,60.00740100012081,312000,1048000,19.974347599986686,15620034.568748968,50.036712267609495, +crayon:cpu:standard,unicode,OK,54.55219800001032,58.899584000073446,216001,660000,8.707001199991282,24807737.479146812,72.28953279904856, +crayon:cpu:standard,mixed,OK,52.35503600010816,56.67740900003082,372501,1397500,24.713216800000737,15072946.715701897,53.92903190076392, +crayon:cpu:lite,english,OK,12.436990999958653,8.398851000038121,144001,740000,8.275760400033505,17400334.596373405,85.27542606692285, +crayon:cpu:lite,code,OK,8.394196999915948,8.967315000063536,556000,1048000,19.186440499993296,28978798.855378844,52.091511377219724, +crayon:cpu:lite,unicode,OK,8.202684000025329,7.951495000042996,474000,660000,11.630868800034477,40753619.368365236,54.11676974864158, +crayon:cpu:lite,mixed,OK,8.248566000020219,7.9241519999868615,640000,1397500,25.899429999958556,24710968.54259048,51.45904203991621, +crayon:cpu:standard,english,OK,55.804826999974466,55.21592800005237,136001,740000,8.24770939998416,16489547.995018011,85.5654533781198, +crayon:cpu:standard,code,OK,53.25067099988701,56.176632999950016,312000,1048000,19.01694320004026,16406422.24767961,52.555801060163766, +crayon:cpu:standard,unicode,OK,51.37607200003913,57.08421299982547,216001,660000,8.193823400006295,26361441.961250234,76.81701424363642, +crayon:cpu:standard,mixed,OK,50.87619200003246,55.96328999990874,372501,1397500,23.42254420000245,15903524.263600752,56.90072973275017, +crayon:cpu:lite,english,OK,12.681803000077707,8.339449000004606,144001,740000,7.946261100028097,18121856.076374188,88.81145299115954, +crayon:cpu:lite,code,OK,8.489855999869178,8.429255999999441,556000,1048000,17.839499699971384,31166793.315447733,56.02459151897366, +crayon:cpu:lite,unicode,OK,8.425465000073018,7.948933999841756,474000,660000,11.340071300014642,41798678.99061517,55.50450541058876, +crayon:cpu:lite,mixed,OK,8.271361999959481,8.063301999982286,640000,1397500,24.710047899998244,25900394.95633861,53.93594794196368, +crayon:cpu:standard,english,OK,57.399346000011064,57.00687799981097,136001,740000,7.366327199929401,18462524.988206256,95.80337323970467, +crayon:cpu:standard,code,OK,52.80974899983448,55.450136999979804,312000,1048000,17.28713269999389,18048105.803000532,57.814716930709004, +crayon:cpu:standard,unicode,OK,49.911187999896356,55.01079300006495,216001,660000,7.827426800008651,27595403.38336492,80.41276717240324, +crayon:cpu:standard,mixed,OK,50.642335000020466,55.71895299999596,372501,1397500,22.138317199937774,16826075.65136193,60.20149793415556, +crayon:cpu:lite,english,OK,12.416612000151872,8.006669999986116,144001,740000,7.414434900101696,19421709.400675282,95.18176417341601, +crayon:cpu:lite,code,OK,8.184946000028503,8.040703000006033,556000,1048000,17.015092099973117,32676872.786417566,58.7390698634731, +crayon:cpu:lite,unicode,OK,8.119394000004831,10.240795999834518,474000,660000,11.250017599991224,42133267.41821006,55.94880570041206, +crayon:cpu:lite,mixed,OK,8.207257000094614,8.105700999976762,640000,1397500,24.114371299992854,26540190.16453436,55.26828133305426, +crayon:cpu:standard,english,OK,56.6799999999148,56.85996900001555,136001,740000,7.276114200021766,18691432.852935866,96.99119265314911, +crayon:cpu:standard,code,OK,53.8767259999986,56.392980999817155,312000,1048000,16.931076499963638,18427652.843023308,59.030545612140884, +crayon:cpu:standard,unicode,OK,52.14681500001461,56.97598899996592,216001,660000,9.631623399945966,22426229.829668358,65.34984007282263, +crayon:cpu:standard,mixed,OK,64.50170099992647,73.0448100000558,372501,1397500,22.081350899952668,16869484.194501817,60.35680802394165, +crayon:cpu:lite,english,OK,12.613476000069568,8.439872000053583,144001,740000,7.526745599921014,19131907.421118516,93.76150485915596, +crayon:cpu:lite,code,OK,8.35724799981108,7.943710999825271,556000,1048000,17.169378300013705,32383234.284001783,58.21123316928443, +crayon:cpu:lite,unicode,OK,8.207450000099925,7.842661999802658,474000,660000,11.432291400001304,41461504.38397205,55.056770931158496, +crayon:cpu:lite,mixed,OK,8.383187999925212,8.207669000057649,640000,1397500,24.294355600022755,26343567.63920096,54.85882725683352, +crayon:cpu:standard,english,OK,56.950104999941686,56.34063200000128,136001,740000,7.244046000005255,18774176.751486853,97.42055670824192, +crayon:cpu:standard,code,OK,53.80810300016492,56.235243000173796,312000,1048000,17.047571000011885,18301727.55988419,58.627160643182144, +crayon:cpu:standard,unicode,OK,51.413830000001326,56.52497999994921,216001,660000,7.654558999979599,28218608.02177835,82.22878010735857, +crayon:cpu:standard,mixed,OK,50.74901000011778,56.04156000003968,372501,1397500,21.856627100009973,17042931.56924611,60.97738004493503, +crayon:cpu:lite,english,OK,12.375640000072963,8.216795000180355,144001,740000,7.549861500001498,19073330.020685997,93.47442918528836, +crayon:cpu:lite,code,OK,8.370037999839042,8.492414999864195,556000,1048000,17.04479079999146,32619936.878326405,58.63672340256888, +crayon:cpu:lite,unicode,OK,8.374210999818388,8.294411999941076,474000,660000,11.23087790003865,42205071.07448553,56.04415384357076, +crayon:cpu:lite,mixed,OK,8.168038000121669,7.773352999947747,640000,1397500,23.87245669999629,26809138.583550114,55.8283495463599, +crayon:cpu:standard,english,OK,56.393689000060476,56.11730300006457,136001,740000,7.189235799955895,18917309.681348097,98.16328380061681, +crayon:cpu:standard,code,OK,55.14521199984301,56.80338499996651,312000,1048000,16.976206699951035,18378664.062855687,58.873616542182695, +crayon:cpu:standard,unicode,OK,50.930590000007214,55.950826000071174,216001,660000,7.773502399959398,27786831.326009266,80.9705865442857, +crayon:cpu:standard,mixed,OK,50.68236399984016,56.41960100001597,372501,1397500,22.1512556000107,16816247.65324003,60.16633464231665, +crayon:cpu:lite,english,OK,12.926781999794912,8.272389000012481,144001,740000,7.4808676000657215,19249237.882346015,94.33651708184557, +crayon:cpu:lite,code,OK,8.45347400013452,8.137007999948764,556000,1048000,16.98900330000015,32727052.32801945,58.82927126123645, +crayon:cpu:lite,unicode,OK,8.421383000040805,8.281079999960639,474000,660000,11.385441799961882,41632113.03768528,55.28332232397274, +crayon:cpu:lite,mixed,OK,8.327604999976757,8.21830199993201,640000,1397500,23.69762730002094,27006923.178314753,56.24022355928252, +crayon:cpu:standard,english,OK,59.05910399997083,58.100684000010006,136001,740000,7.118251499991857,19105956.00621242,99.1421831810006, +crayon:cpu:standard,code,OK,54.08621699984906,55.60879500012561,312000,1048000,16.766940600018643,18608045.882840015,59.60841082675743, +crayon:cpu:standard,unicode,OK,51.236133999964295,56.99880799988932,216001,660000,7.7092513000025065,28018414.706487745,81.64541851527403, +crayon:cpu:standard,mixed,OK,49.82170999983282,55.206914000109464,372501,1397500,21.801854500017726,17085748.370612103,61.130572960050294, +crayon:cpu:lite,english,OK,12.190522999844688,8.098015000086889,144001,740000,7.3088084999881175,19702390.615410723,96.55732451353354, +crayon:cpu:lite,code,OK,8.321124999838503,8.069172999967122,556000,1048000,17.128919400033737,32459724.22515485,58.34872943542378, +crayon:cpu:lite,unicode,OK,8.535713000128453,8.166854999899442,474000,660000,11.439660299970456,41434796.80084768,55.02130590623837, +crayon:cpu:lite,mixed,OK,8.366541999976107,8.283507999976791,640000,1397500,24.293525899997803,26344467.35457441,54.86070085766574, +crayon:cpu:standard,english,OK,57.363589000033244,56.69689699993796,136001,740000,7.236277000015434,18794333.052716188,97.52514920851148, +crayon:cpu:standard,code,OK,53.46871999995528,54.877045000012004,312000,1048000,16.9864471999972,18367584.246813625,58.83812381872971, +crayon:cpu:standard,unicode,OK,50.751881999985926,55.154549000008046,216001,660000,7.6573333000396815,28208384.242446475,82.19898810266797, +crayon:cpu:standard,mixed,OK,50.123076999852856,55.0747180000144,372501,1397500,21.902375399963603,17007333.368992437,60.85001434045137, +crayon:cpu:lite,english,OK,12.418854000088686,8.39753200011728,144001,740000,7.652865499994732,18816611.895256635,92.21630696908377, +crayon:cpu:lite,code,OK,8.422789000178454,7.939616999919963,556000,1048000,17.000992600014797,32703972.825652312,58.78778416695976, +crayon:cpu:lite,unicode,OK,8.08434100008526,7.726747000106116,474000,660000,11.228830199956974,42212767.6311123,56.05437410840328, +crayon:cpu:lite,mixed,OK,8.228894000012588,8.049411000001783,640000,1397500,23.87932870003624,26801423.44198431,55.81228324796717, +crayon:cpu:standard,english,OK,57.396696000068914,57.40757900002791,136001,740000,7.397213799981728,18385435.878619045,95.40335229223308, +crayon:cpu:standard,code,OK,54.45600699999886,55.53728500012767,312000,1048000,16.702699099982965,18679615.679619003,59.837675193153025, +crayon:cpu:standard,unicode,OK,50.2353619999667,54.39181100018686,216001,660000,7.571766700016269,28527159.982297905,83.12789785596175, +crayon:cpu:standard,mixed,OK,49.25932800006194,55.9337869999581,372501,1397500,21.99373309999828,16936688.20597032,60.597255187107756, +crayon:cpu:lite,english,OK,12.266837999959534,8.24019199990289,144001,740000,7.97089010000036,18065861.92927105,88.53703730535604, +crayon:cpu:lite,code,OK,9.197482999979911,9.25707299984424,556000,1048000,17.070629699992423,32570561.822933037,58.54796812763115, +crayon:cpu:lite,unicode,OK,8.248180999999022,8.046429999922111,474000,660000,11.251252000033674,42128644.88312779,55.94266743169926, +crayon:cpu:lite,mixed,OK,8.281324999870776,8.082607999995162,640000,1397500,23.73838900005012,26960549.00771273,56.143652257738324, +crayon:cpu:standard,english,OK,56.84132600003977,56.485478000013245,136001,740000,7.21978010001294,18837277.329230048,97.74799015545642, +crayon:cpu:standard,code,OK,52.64668600011646,55.48405200011075,312000,1048000,16.53436149999834,18869794.276605804,60.446887144317635, +crayon:cpu:standard,unicode,OK,49.47402999982842,54.24425399996835,216001,660000,7.538081999973656,28654636.55088322,83.49936347605727, +crayon:cpu:standard,mixed,OK,49.328699999932724,54.43467200007035,372501,1397500,22.05378749997635,16890568.116719157,60.43224353998893, diff --git a/benchmark_results/20260317_053838/benchmark_results.json b/benchmark_results/20260317_053838/benchmark_results.json new file mode 100644 index 0000000..8951826 --- /dev/null +++ b/benchmark_results/20260317_053838/benchmark_results.json @@ -0,0 +1,1042 @@ +[ + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 13.410531000090486, + "warm_load_time_ms": 9.785870999849067, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 8.480945400037854, + "tokens_per_sec": 16979357.041887958, + "mb_per_sec": 83.21230250314724, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.735390000083498, + "warm_load_time_ms": 8.888432000048851, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 19.00110249998761, + "tokens_per_sec": 29261459.960039824, + "mb_per_sec": 52.599615395706735, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 12.106269000014436, + "warm_load_time_ms": 8.023800000046322, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.997211600009905, + "tokens_per_sec": 39509180.61656999, + "mb_per_sec": 52.464278351780116, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 12.21738499998537, + "warm_load_time_ms": 8.244456000056744, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 27.57976560003499, + "tokens_per_sec": 23205418.395549674, + "mb_per_sec": 48.32382829156617, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 58.660766000002695, + "warm_load_time_ms": 56.090405000077226, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 8.652120299984745, + "tokens_per_sec": 15718805.944045855, + "mb_per_sec": 81.56601730813536, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 56.99175500012643, + "warm_load_time_ms": 60.00740100012081, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 19.974347599986686, + "tokens_per_sec": 15620034.568748968, + "mb_per_sec": 50.036712267609495, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 54.55219800001032, + "warm_load_time_ms": 58.899584000073446, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 8.707001199991282, + "tokens_per_sec": 24807737.479146812, + "mb_per_sec": 72.28953279904856, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 52.35503600010816, + "warm_load_time_ms": 56.67740900003082, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 24.713216800000737, + "tokens_per_sec": 15072946.715701897, + "mb_per_sec": 53.92903190076392, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.436990999958653, + "warm_load_time_ms": 8.398851000038121, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 8.275760400033505, + "tokens_per_sec": 17400334.596373405, + "mb_per_sec": 85.27542606692285, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.394196999915948, + "warm_load_time_ms": 8.967315000063536, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 19.186440499993296, + "tokens_per_sec": 28978798.855378844, + "mb_per_sec": 52.091511377219724, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.202684000025329, + "warm_load_time_ms": 7.951495000042996, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.630868800034477, + "tokens_per_sec": 40753619.368365236, + "mb_per_sec": 54.11676974864158, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.248566000020219, + "warm_load_time_ms": 7.9241519999868615, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 25.899429999958556, + "tokens_per_sec": 24710968.54259048, + "mb_per_sec": 51.45904203991621, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 55.804826999974466, + "warm_load_time_ms": 55.21592800005237, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 8.24770939998416, + "tokens_per_sec": 16489547.995018011, + "mb_per_sec": 85.5654533781198, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 53.25067099988701, + "warm_load_time_ms": 56.176632999950016, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 19.01694320004026, + "tokens_per_sec": 16406422.24767961, + "mb_per_sec": 52.555801060163766, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 51.37607200003913, + "warm_load_time_ms": 57.08421299982547, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 8.193823400006295, + "tokens_per_sec": 26361441.961250234, + "mb_per_sec": 76.81701424363642, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 50.87619200003246, + "warm_load_time_ms": 55.96328999990874, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 23.42254420000245, + "tokens_per_sec": 15903524.263600752, + "mb_per_sec": 56.90072973275017, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.681803000077707, + "warm_load_time_ms": 8.339449000004606, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.946261100028097, + "tokens_per_sec": 18121856.076374188, + "mb_per_sec": 88.81145299115954, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.489855999869178, + "warm_load_time_ms": 8.429255999999441, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 17.839499699971384, + "tokens_per_sec": 31166793.315447733, + "mb_per_sec": 56.02459151897366, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.425465000073018, + "warm_load_time_ms": 7.948933999841756, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.340071300014642, + "tokens_per_sec": 41798678.99061517, + "mb_per_sec": 55.50450541058876, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.271361999959481, + "warm_load_time_ms": 8.063301999982286, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 24.710047899998244, + "tokens_per_sec": 25900394.95633861, + "mb_per_sec": 53.93594794196368, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 57.399346000011064, + "warm_load_time_ms": 57.00687799981097, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.366327199929401, + "tokens_per_sec": 18462524.988206256, + "mb_per_sec": 95.80337323970467, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 52.80974899983448, + "warm_load_time_ms": 55.450136999979804, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 17.28713269999389, + "tokens_per_sec": 18048105.803000532, + "mb_per_sec": 57.814716930709004, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 49.911187999896356, + "warm_load_time_ms": 55.01079300006495, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.827426800008651, + "tokens_per_sec": 27595403.38336492, + "mb_per_sec": 80.41276717240324, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 50.642335000020466, + "warm_load_time_ms": 55.71895299999596, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 22.138317199937774, + "tokens_per_sec": 16826075.65136193, + "mb_per_sec": 60.20149793415556, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.416612000151872, + "warm_load_time_ms": 8.006669999986116, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.414434900101696, + "tokens_per_sec": 19421709.400675282, + "mb_per_sec": 95.18176417341601, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.184946000028503, + "warm_load_time_ms": 8.040703000006033, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 17.015092099973117, + "tokens_per_sec": 32676872.786417566, + "mb_per_sec": 58.7390698634731, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.119394000004831, + "warm_load_time_ms": 10.240795999834518, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.250017599991224, + "tokens_per_sec": 42133267.41821006, + "mb_per_sec": 55.94880570041206, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.207257000094614, + "warm_load_time_ms": 8.105700999976762, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 24.114371299992854, + "tokens_per_sec": 26540190.16453436, + "mb_per_sec": 55.26828133305426, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 56.6799999999148, + "warm_load_time_ms": 56.85996900001555, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.276114200021766, + "tokens_per_sec": 18691432.852935866, + "mb_per_sec": 96.99119265314911, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 53.8767259999986, + "warm_load_time_ms": 56.392980999817155, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 16.931076499963638, + "tokens_per_sec": 18427652.843023308, + "mb_per_sec": 59.030545612140884, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 52.14681500001461, + "warm_load_time_ms": 56.97598899996592, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 9.631623399945966, + "tokens_per_sec": 22426229.829668358, + "mb_per_sec": 65.34984007282263, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 64.50170099992647, + "warm_load_time_ms": 73.0448100000558, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 22.081350899952668, + "tokens_per_sec": 16869484.194501817, + "mb_per_sec": 60.35680802394165, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.613476000069568, + "warm_load_time_ms": 8.439872000053583, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.526745599921014, + "tokens_per_sec": 19131907.421118516, + "mb_per_sec": 93.76150485915596, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.35724799981108, + "warm_load_time_ms": 7.943710999825271, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 17.169378300013705, + "tokens_per_sec": 32383234.284001783, + "mb_per_sec": 58.21123316928443, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.207450000099925, + "warm_load_time_ms": 7.842661999802658, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.432291400001304, + "tokens_per_sec": 41461504.38397205, + "mb_per_sec": 55.056770931158496, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.383187999925212, + "warm_load_time_ms": 8.207669000057649, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 24.294355600022755, + "tokens_per_sec": 26343567.63920096, + "mb_per_sec": 54.85882725683352, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 56.950104999941686, + "warm_load_time_ms": 56.34063200000128, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.244046000005255, + "tokens_per_sec": 18774176.751486853, + "mb_per_sec": 97.42055670824192, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 53.80810300016492, + "warm_load_time_ms": 56.235243000173796, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 17.047571000011885, + "tokens_per_sec": 18301727.55988419, + "mb_per_sec": 58.627160643182144, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 51.413830000001326, + "warm_load_time_ms": 56.52497999994921, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.654558999979599, + "tokens_per_sec": 28218608.02177835, + "mb_per_sec": 82.22878010735857, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 50.74901000011778, + "warm_load_time_ms": 56.04156000003968, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 21.856627100009973, + "tokens_per_sec": 17042931.56924611, + "mb_per_sec": 60.97738004493503, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.375640000072963, + "warm_load_time_ms": 8.216795000180355, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.549861500001498, + "tokens_per_sec": 19073330.020685997, + "mb_per_sec": 93.47442918528836, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.370037999839042, + "warm_load_time_ms": 8.492414999864195, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 17.04479079999146, + "tokens_per_sec": 32619936.878326405, + "mb_per_sec": 58.63672340256888, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.374210999818388, + "warm_load_time_ms": 8.294411999941076, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.23087790003865, + "tokens_per_sec": 42205071.07448553, + "mb_per_sec": 56.04415384357076, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.168038000121669, + "warm_load_time_ms": 7.773352999947747, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 23.87245669999629, + "tokens_per_sec": 26809138.583550114, + "mb_per_sec": 55.8283495463599, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 56.393689000060476, + "warm_load_time_ms": 56.11730300006457, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.189235799955895, + "tokens_per_sec": 18917309.681348097, + "mb_per_sec": 98.16328380061681, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 55.14521199984301, + "warm_load_time_ms": 56.80338499996651, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 16.976206699951035, + "tokens_per_sec": 18378664.062855687, + "mb_per_sec": 58.873616542182695, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 50.930590000007214, + "warm_load_time_ms": 55.950826000071174, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.773502399959398, + "tokens_per_sec": 27786831.326009266, + "mb_per_sec": 80.9705865442857, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 50.68236399984016, + "warm_load_time_ms": 56.41960100001597, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 22.1512556000107, + "tokens_per_sec": 16816247.65324003, + "mb_per_sec": 60.16633464231665, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.926781999794912, + "warm_load_time_ms": 8.272389000012481, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.4808676000657215, + "tokens_per_sec": 19249237.882346015, + "mb_per_sec": 94.33651708184557, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.45347400013452, + "warm_load_time_ms": 8.137007999948764, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 16.98900330000015, + "tokens_per_sec": 32727052.32801945, + "mb_per_sec": 58.82927126123645, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.421383000040805, + "warm_load_time_ms": 8.281079999960639, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.385441799961882, + "tokens_per_sec": 41632113.03768528, + "mb_per_sec": 55.28332232397274, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.327604999976757, + "warm_load_time_ms": 8.21830199993201, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 23.69762730002094, + "tokens_per_sec": 27006923.178314753, + "mb_per_sec": 56.24022355928252, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 59.05910399997083, + "warm_load_time_ms": 58.100684000010006, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.118251499991857, + "tokens_per_sec": 19105956.00621242, + "mb_per_sec": 99.1421831810006, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 54.08621699984906, + "warm_load_time_ms": 55.60879500012561, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 16.766940600018643, + "tokens_per_sec": 18608045.882840015, + "mb_per_sec": 59.60841082675743, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 51.236133999964295, + "warm_load_time_ms": 56.99880799988932, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.7092513000025065, + "tokens_per_sec": 28018414.706487745, + "mb_per_sec": 81.64541851527403, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 49.82170999983282, + "warm_load_time_ms": 55.206914000109464, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 21.801854500017726, + "tokens_per_sec": 17085748.370612103, + "mb_per_sec": 61.130572960050294, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.190522999844688, + "warm_load_time_ms": 8.098015000086889, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.3088084999881175, + "tokens_per_sec": 19702390.615410723, + "mb_per_sec": 96.55732451353354, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.321124999838503, + "warm_load_time_ms": 8.069172999967122, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 17.128919400033737, + "tokens_per_sec": 32459724.22515485, + "mb_per_sec": 58.34872943542378, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.535713000128453, + "warm_load_time_ms": 8.166854999899442, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.439660299970456, + "tokens_per_sec": 41434796.80084768, + "mb_per_sec": 55.02130590623837, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.366541999976107, + "warm_load_time_ms": 8.283507999976791, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 24.293525899997803, + "tokens_per_sec": 26344467.35457441, + "mb_per_sec": 54.86070085766574, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 57.363589000033244, + "warm_load_time_ms": 56.69689699993796, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.236277000015434, + "tokens_per_sec": 18794333.052716188, + "mb_per_sec": 97.52514920851148, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 53.46871999995528, + "warm_load_time_ms": 54.877045000012004, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 16.9864471999972, + "tokens_per_sec": 18367584.246813625, + "mb_per_sec": 58.83812381872971, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 50.751881999985926, + "warm_load_time_ms": 55.154549000008046, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.6573333000396815, + "tokens_per_sec": 28208384.242446475, + "mb_per_sec": 82.19898810266797, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 50.123076999852856, + "warm_load_time_ms": 55.0747180000144, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 21.902375399963603, + "tokens_per_sec": 17007333.368992437, + "mb_per_sec": 60.85001434045137, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.418854000088686, + "warm_load_time_ms": 8.39753200011728, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.652865499994732, + "tokens_per_sec": 18816611.895256635, + "mb_per_sec": 92.21630696908377, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 8.422789000178454, + "warm_load_time_ms": 7.939616999919963, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 17.000992600014797, + "tokens_per_sec": 32703972.825652312, + "mb_per_sec": 58.78778416695976, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.08434100008526, + "warm_load_time_ms": 7.726747000106116, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.228830199956974, + "tokens_per_sec": 42212767.6311123, + "mb_per_sec": 56.05437410840328, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.228894000012588, + "warm_load_time_ms": 8.049411000001783, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 23.87932870003624, + "tokens_per_sec": 26801423.44198431, + "mb_per_sec": 55.81228324796717, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 57.396696000068914, + "warm_load_time_ms": 57.40757900002791, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.397213799981728, + "tokens_per_sec": 18385435.878619045, + "mb_per_sec": 95.40335229223308, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 54.45600699999886, + "warm_load_time_ms": 55.53728500012767, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 16.702699099982965, + "tokens_per_sec": 18679615.679619003, + "mb_per_sec": 59.837675193153025, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 50.2353619999667, + "warm_load_time_ms": 54.39181100018686, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.571766700016269, + "tokens_per_sec": 28527159.982297905, + "mb_per_sec": 83.12789785596175, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 49.25932800006194, + "warm_load_time_ms": 55.9337869999581, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 21.99373309999828, + "tokens_per_sec": 16936688.20597032, + "mb_per_sec": 60.597255187107756, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "status": "OK", + "cold_load_time_ms": 12.266837999959534, + "warm_load_time_ms": 8.24019199990289, + "tokens_produced": 144001, + "bytes_processed": 740000, + "avg_time_ms": 7.97089010000036, + "tokens_per_sec": 18065861.92927105, + "mb_per_sec": 88.53703730535604, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "code", + "status": "OK", + "cold_load_time_ms": 9.197482999979911, + "warm_load_time_ms": 9.25707299984424, + "tokens_produced": 556000, + "bytes_processed": 1048000, + "avg_time_ms": 17.070629699992423, + "tokens_per_sec": 32570561.822933037, + "mb_per_sec": 58.54796812763115, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 8.248180999999022, + "warm_load_time_ms": 8.046429999922111, + "tokens_produced": 474000, + "bytes_processed": 660000, + "avg_time_ms": 11.251252000033674, + "tokens_per_sec": 42128644.88312779, + "mb_per_sec": 55.94266743169926, + "notes": "" + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 8.281324999870776, + "warm_load_time_ms": 8.082607999995162, + "tokens_produced": 640000, + "bytes_processed": 1397500, + "avg_time_ms": 23.73838900005012, + "tokens_per_sec": 26960549.00771273, + "mb_per_sec": 56.143652257738324, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "status": "OK", + "cold_load_time_ms": 56.84132600003977, + "warm_load_time_ms": 56.485478000013245, + "tokens_produced": 136001, + "bytes_processed": 740000, + "avg_time_ms": 7.21978010001294, + "tokens_per_sec": 18837277.329230048, + "mb_per_sec": 97.74799015545642, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "status": "OK", + "cold_load_time_ms": 52.64668600011646, + "warm_load_time_ms": 55.48405200011075, + "tokens_produced": 312000, + "bytes_processed": 1048000, + "avg_time_ms": 16.53436149999834, + "tokens_per_sec": 18869794.276605804, + "mb_per_sec": 60.446887144317635, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "status": "OK", + "cold_load_time_ms": 49.47402999982842, + "warm_load_time_ms": 54.24425399996835, + "tokens_produced": 216001, + "bytes_processed": 660000, + "avg_time_ms": 7.538081999973656, + "tokens_per_sec": 28654636.55088322, + "mb_per_sec": 83.49936347605727, + "notes": "" + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "status": "OK", + "cold_load_time_ms": 49.328699999932724, + "warm_load_time_ms": 54.43467200007035, + "tokens_produced": 372501, + "bytes_processed": 1397500, + "avg_time_ms": 22.05378749997635, + "tokens_per_sec": 16890568.116719157, + "mb_per_sec": 60.43224353998893, + "notes": "" + } +] \ No newline at end of file diff --git a/benchmark_results/20260317_053838/benchmark_summary.csv b/benchmark_results/20260317_053838/benchmark_summary.csv new file mode 100644 index 0000000..34db2de --- /dev/null +++ b/benchmark_results/20260317_053838/benchmark_summary.csv @@ -0,0 +1,9 @@ +impl,case,n,tokens_per_sec_mean,tokens_per_sec_std,cold_load_time_ms_mean,cold_load_time_ms_std,warm_load_time_ms_mean,warm_load_time_ms_std,mb_per_sec_mean,mb_per_sec_std,tokens_produced_mean,tokens_produced_std +crayon:cpu:lite,code,10,31754840.728137184,1463286.9241903918,8.492654599967864,0.2846632249461966,8.416470299948742,0.47487165302873435,57.081649771847765,2.630365317762379,556000.0,0.0 +crayon:cpu:lite,english,10,18596259.68793998,909538.0165265289,12.573805000010907,0.3629786094116921,8.419563600023139,0.49926227356328895,91.13640656489089,4.457456921519721,144001.0,0.0 +crayon:cpu:lite,mixed,10,26062304.12643504,1211263.2030730038,8.67001619999428,1.2482779745781352,8.09524619999138,0.15697362395806927,54.27311363323476,2.522379646144097,640000.0,0.0 +crayon:cpu:lite,unicode,10,41526964.42049911,845048.0825815537,8.672509100028947,1.2152478046731008,8.252321099939763,0.721505942514216,55.143695375646544,1.1221401490316623,474000.0,0.0 +crayon:cpu:standard,code,10,17970764.717107076,1071761.9519094536,54.05398459997741,1.2733276807154361,56.25729570003841,1.4329603439418621,57.56696500389457,3.4332474855310093,312000.0,0.0 +crayon:cpu:standard,english,10,18217680.047981866,1147363.1036781135,57.254944800001795,0.9845046618087464,56.63217530000111,0.7905787073616826,94.53285519251692,5.953749865381758,136001.0,0.0 +crayon:cpu:standard,mixed,10,16645154.810994655,646608.6486216456,51.833945299972584,4.539518942642857,57.45157140001993,5.518773323467621,59.55418683064613,2.313481172364295,372501.0,0.0 +crayon:cpu:standard,unicode,10,27060484.748333327,2003401.990129244,51.20281009997143,1.4163948040007757,56.123580700000275,1.4537618289387846,78.85401888895161,5.837896099830383,216001.0,0.0 diff --git a/benchmark_results/20260317_053838/benchmark_summary.json b/benchmark_results/20260317_053838/benchmark_summary.json new file mode 100644 index 0000000..52ca2d2 --- /dev/null +++ b/benchmark_results/20260317_053838/benchmark_summary.json @@ -0,0 +1,122 @@ +[ + { + "impl": "crayon:cpu:lite", + "case": "code", + "n": 10, + "tokens_per_sec_mean": 31754840.728137184, + "tokens_per_sec_std": 1463286.9241903918, + "cold_load_time_ms_mean": 8.492654599967864, + "cold_load_time_ms_std": 0.2846632249461966, + "warm_load_time_ms_mean": 8.416470299948742, + "warm_load_time_ms_std": 0.47487165302873435, + "mb_per_sec_mean": 57.081649771847765, + "mb_per_sec_std": 2.630365317762379, + "tokens_produced_mean": 556000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:lite", + "case": "english", + "n": 10, + "tokens_per_sec_mean": 18596259.68793998, + "tokens_per_sec_std": 909538.0165265289, + "cold_load_time_ms_mean": 12.573805000010907, + "cold_load_time_ms_std": 0.3629786094116921, + "warm_load_time_ms_mean": 8.419563600023139, + "warm_load_time_ms_std": 0.49926227356328895, + "mb_per_sec_mean": 91.13640656489089, + "mb_per_sec_std": 4.457456921519721, + "tokens_produced_mean": 144001.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:lite", + "case": "mixed", + "n": 10, + "tokens_per_sec_mean": 26062304.12643504, + "tokens_per_sec_std": 1211263.2030730038, + "cold_load_time_ms_mean": 8.67001619999428, + "cold_load_time_ms_std": 1.2482779745781352, + "warm_load_time_ms_mean": 8.09524619999138, + "warm_load_time_ms_std": 0.15697362395806927, + "mb_per_sec_mean": 54.27311363323476, + "mb_per_sec_std": 2.522379646144097, + "tokens_produced_mean": 640000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:lite", + "case": "unicode", + "n": 10, + "tokens_per_sec_mean": 41526964.42049911, + "tokens_per_sec_std": 845048.0825815537, + "cold_load_time_ms_mean": 8.672509100028947, + "cold_load_time_ms_std": 1.2152478046731008, + "warm_load_time_ms_mean": 8.252321099939763, + "warm_load_time_ms_std": 0.721505942514216, + "mb_per_sec_mean": 55.143695375646544, + "mb_per_sec_std": 1.1221401490316623, + "tokens_produced_mean": 474000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "code", + "n": 10, + "tokens_per_sec_mean": 17970764.717107076, + "tokens_per_sec_std": 1071761.9519094536, + "cold_load_time_ms_mean": 54.05398459997741, + "cold_load_time_ms_std": 1.2733276807154361, + "warm_load_time_ms_mean": 56.25729570003841, + "warm_load_time_ms_std": 1.4329603439418621, + "mb_per_sec_mean": 57.56696500389457, + "mb_per_sec_std": 3.4332474855310093, + "tokens_produced_mean": 312000.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "english", + "n": 10, + "tokens_per_sec_mean": 18217680.047981866, + "tokens_per_sec_std": 1147363.1036781135, + "cold_load_time_ms_mean": 57.254944800001795, + "cold_load_time_ms_std": 0.9845046618087464, + "warm_load_time_ms_mean": 56.63217530000111, + "warm_load_time_ms_std": 0.7905787073616826, + "mb_per_sec_mean": 94.53285519251692, + "mb_per_sec_std": 5.953749865381758, + "tokens_produced_mean": 136001.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "mixed", + "n": 10, + "tokens_per_sec_mean": 16645154.810994655, + "tokens_per_sec_std": 646608.6486216456, + "cold_load_time_ms_mean": 51.833945299972584, + "cold_load_time_ms_std": 4.539518942642857, + "warm_load_time_ms_mean": 57.45157140001993, + "warm_load_time_ms_std": 5.518773323467621, + "mb_per_sec_mean": 59.55418683064613, + "mb_per_sec_std": 2.313481172364295, + "tokens_produced_mean": 372501.0, + "tokens_produced_std": 0.0 + }, + { + "impl": "crayon:cpu:standard", + "case": "unicode", + "n": 10, + "tokens_per_sec_mean": 27060484.748333327, + "tokens_per_sec_std": 2003401.990129244, + "cold_load_time_ms_mean": 51.20281009997143, + "cold_load_time_ms_std": 1.4163948040007757, + "warm_load_time_ms_mean": 56.123580700000275, + "warm_load_time_ms_std": 1.4537618289387846, + "mb_per_sec_mean": 78.85401888895161, + "mb_per_sec_std": 5.837896099830383, + "tokens_produced_mean": 216001.0, + "tokens_produced_std": 0.0 + } +] \ No newline at end of file diff --git a/benchmark_results/20260317_053838/metadata.json b/benchmark_results/20260317_053838/metadata.json new file mode 100644 index 0000000..fc05ea8 --- /dev/null +++ b/benchmark_results/20260317_053838/metadata.json @@ -0,0 +1,22 @@ +{ + "timestamp": "2026-03-17T05:38:38", + "cwd": "/app", + "device_arg": "cpu", + "platform": "Linux-6.8.0-x86_64-with-glibc2.39", + "python": "3.12.13 (main, Mar 6 2026, 16:37:31) [GCC 13.3.0]", + "processor": "x86_64", + "cpu_count_logical": 4, + "ram_total_bytes": null, + "nvidia_smi": null, + "rocm_smi": null, + "versions": { + "tiktoken": null, + "transformers": null, + "matplotlib": null + }, + "backends": { + "torch": "2.10.0+cu128", + "torch_cuda_is_available": false, + "torch_cuda_device_count": 0 + } +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 75669b9..f7d2b52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68.0", "wheel"] +requires = ["setuptools>=68.0", "wheel", "torch", "numpy"] build-backend = "setuptools.build_meta" [project] diff --git a/setup.py b/setup.py index daf9b90..9007194 100644 --- a/setup.py +++ b/setup.py @@ -303,14 +303,64 @@ def _build_hip_extension(self, ext): # SETUP ENTRY POINT # ============================================================================ -setup( - name="xerv-crayon", - version=VERSION, - packages=find_packages("src"), - package_dir={"": "src"}, - include_package_data=True, - ext_modules=ext_modules, - cmdclass=cmdclass, - python_requires=">=3.10", - zip_safe=False, -) +def safe_setup(): + import sys + try: + setup( + name="xerv-crayon", + version=VERSION, + packages=find_packages("src"), + package_dir={"": "src"}, + include_package_data=True, + ext_modules=ext_modules, + cmdclass=cmdclass, + python_requires=">=3.10", + zip_safe=False, + ) + except SystemExit as e: + if e.code != 0: + log(f"Initial setup with GPU extensions failed with SystemExit: {e.code}") + log("Falling back to CPU-only build.") + + # Filter out GPU extensions + cpu_only_ext_modules = [ + ext for ext in ext_modules + if ext.name in ["crayon.c_ext.crayon_cpu", "crayon.c_ext.crayon_trainer", "crayon.c_ext.crayon_compiler"] + ] + + setup( + name="xerv-crayon", + version=VERSION, + packages=find_packages("src"), + package_dir={"": "src"}, + include_package_data=True, + ext_modules=cpu_only_ext_modules, + cmdclass={}, + python_requires=">=3.10", + zip_safe=False, + ) + else: + raise + except Exception as e: + log(f"Initial setup with GPU extensions failed: {e}") + log("Falling back to CPU-only build.") + + # Filter out GPU extensions + cpu_only_ext_modules = [ + ext for ext in ext_modules + if ext.name in ["crayon.c_ext.crayon_cpu", "crayon.c_ext.crayon_trainer", "crayon.c_ext.crayon_compiler"] + ] + + setup( + name="xerv-crayon", + version=VERSION, + packages=find_packages("src"), + package_dir={"": "src"}, + include_package_data=True, + ext_modules=cpu_only_ext_modules, + cmdclass={}, + python_requires=">=3.10", + zip_safe=False, + ) + +safe_setup() diff --git a/test_cuda.py b/test_cuda.py new file mode 100644 index 0000000..61286bb --- /dev/null +++ b/test_cuda.py @@ -0,0 +1,5 @@ +from crayon import CrayonVocab +tokenizer = CrayonVocab(device="cuda") +tokenizer.load_profile("standard") +tokens = tokenizer.tokenize("someunknownPlace") +print(tokens) diff --git a/test_rocm.py b/test_rocm.py new file mode 100644 index 0000000..57b8cad --- /dev/null +++ b/test_rocm.py @@ -0,0 +1,8 @@ +from crayon import CrayonVocab +try: + tokenizer = CrayonVocab(device="rocm") + tokenizer.load_profile("standard") + tokens = tokenizer.tokenize("someunknownPlace") + print(tokens) +except Exception as e: + print(e)