LEANN/research/micro/data/transformer-batching-microbenchmarks.csv

1	n	d	seqlen	bs	latency	h	flop	io	intensity	throughput	series
2	3	256	256	2048	0.009623501679245285	768	618475290624	167.48502132816208	3692720015.912285	64267177503366.266	dense
3	3	256	256	1024	0.004853848615384615	768	309237645312	166.15392854317415	1861151572.059558	63709783682138.234	dense
4	3	256	256	512	0.0024687246971962615	768	154618822656	163.57953256539062	945221081.3366361	62631051097597.516	dense
5	3	256	256	256	0.0012845360838052097	768	77309411328	157.64931990085577	490388486.1451936	60184694149645.54	dense
6	3	256	256	128	0.0006901147179878049	768	38654705664	147.57393422494675	261934506.70684624	56012000116019.945	dense
7	3	256	256	64	0.0003363830693015702	768	19327352832	153.1328437752606	126212981.84970059	57456378146882.51	dense
8	3	256	256	32	0.00018671159748991485	768	9663676416	141.10249365427362	68486928.65540518	51757237075334.75	dense
9	3	256	256	16	0.00012353640857142858	768	4831838208	111.40488993609125	43371868.24359184	39112665358133.98	dense
10	3	256	256	8	9.774760007849294e-05	768	2415919104	76.43260800265766	31608487.09906635	24715891766754.14	dense
11	3	256	256	4	6.672271167474822e-05	768	1207959552	64.82614227498455	18633833.660438772	18104173551704.773	dense
12	3	256	256	2	4.9758770289855074e-05	768	603979776	55.317122669351576	10918495.880745342	12138157202874.861	dense
13	3	256	1	2048	9.785507940251571e-05	768	2415919104	76.34865809334705	31643242.518371396	24688745017132.86	dense
14	3	256	1	1024	6.692813470149253e-05	768	1207959552	64.62717090938949	18691202.70936228	18048606275785.867	dense
15	3	256	1	512	4.9680950036205655e-05	768	603979776	55.40377142534654	10901419.893658841	12157170415618.898	dense
16	3	256	1	256	4.2781118741058655e-05	768	301989888	45.95672244805227	6571179.83862661	7058952568020.829	dense
17	3	256	1	128	5.0662328255350016e-05	768	150994944	31.046026784880404	4863583.512513602	2980418571348.519	dense
18	3	256	1	64	4.475009253945481e-05	768	75497472	30.75426042497223	2454862.219307235	1687090857598.4766	dense
19	3	256	1	32	4.51682671454219e-05	768	37748736	28.29313765537115	1334201.1218340008	835735758435.5786	dense
20	3	256	1	16	5.03585186661834e-05	768	18874368	24.401035466223117	773506.846712577	374799904761.1871	dense
21	3	256	1	8	5.023459565217391e-05	768	9437184	23.972005435021096	393675.19858030166	187862246674.45105	dense
22	3	256	1	4	5.053219391083726e-05	768	4718592	23.58765586356967	200044.97383259286	93377936614.54384	dense
23	3	256	1	2	4.4607398995335484e-05	768	2359296	26.58285456464288	88752.54515134107	52890239133.797226	dense
24	12	256	256	2048	0.14480779847058822	3072	9895604649984	44.620009282941716	221775046868.20184	68336130750540.26	dense
25	12	256	256	1024	0.07254347629166667	3072	4947802324992	44.664248332585096	110777691547.58836	68204648824643.82	dense
26	12	256	256	512	0.036310761444444443	3072	2473901162496	44.876147984203506	55127306456.13385	68131349056975.164	dense
27	12	256	256	256	0.01821551906896552	3072	1236950581248	45.24607467289738	27338295977.947884	67906414116709.98	dense
28	12	256	256	128	0.009229417903030302	3072	618475290624	45.67217092440895	13541622351.335684	67011299859001.46	dense
29	12	256	256	64	0.004754550595394737	3072	309237645312	46.31372736116993	6677019167.566916	65040352207320.695	dense
30	12	256	256	32	0.002405752659340659	3072	154618822656	49.68826015254682	3111777755.5766335	64270456921525.82	dense
31	12	256	256	16	0.0012287219045005488	3072	77309411328	56.323579604557374	1372594069.3184311	62918558743709.18	dense
32	12	256	256	8	0.0006206816149425287	3072	38654705664	70.95456179103653	544781120.315271	62277832520589.78	dense
33	12	256	256	4	0.0003875502697142857	3072	19327352832	81.16954743236613	238110885.71245712	49870569942445.75	dense
34	12	256	256	2	0.00027502018627941914	3072	9663676416	91.50537035282076	105607751.53129694	35138062215483.168	dense
35	12	256	1	2048	0.0006202853873290136	3072	38654705664	70.99988634205897	544433345.6784943	62317614526515.766	dense
36	12	256	1	1024	0.00038721467732724153	3072	19327352832	81.2398957010995	237904697.74985722	49913791918755.53	dense
37	12	256	1	512	0.000274364799	3072	9663676416	91.72395326121995	105356082.81599998	35221998052308.45	dense
38	12	256	1	256	0.00012488918589482266	3072	4831838208	176.31707535146046	27404255.647778228	38689003962834.75	dense
39	12	256	1	128	8.976711102514506e-05	3072	2415919104	227.78088507574267	10606329.425740216	26913187652026.21	dense
40	12	256	1	64	8.715176287471176e-05	3072	1207959552	225.59268282689945	5354604.31102229	13860414432884.701	dense
41	12	256	1	32	8.523013435114503e-05	3072	603979776	226.06539514085782	2671703.8033338524	7086458100741.991	dense
42	12	256	1	16	7.901561645904116e-05	3072	301989888	241.35704882952732	1251216.3595988373	3821901309300.556	dense
43	12	256	1	8	7.827949114210329e-05	3072	150994944	242.37091635608994	622991.1833900034	1928920867994.581	dense
44	12	256	1	4	7.779445951035782e-05	3072	75497472	243.25022783249054	310369.58391664835	970473636235.5986	dense
45	12	256	1	2	7.758845406626506e-05	3072	37748736	243.57933441822672	154975.11761480253	486525172518.07056	dense
46	3	256	256	2048	0.00507974918466899	768	206158430208	475.59810852303485	433471930.42508715	40584371927298.98	qk_init
47	3	256	256	1024	0.0025616677649325623	768	103079215104	471.5519977009198	218595649.27424532	40239103803811.82	qk_init
48	3	256	256	512	0.0013029336670480549	768	51539607552	463.55374128015677	111183672.92143403	39556585922573.38	qk_init
49	3	256	256	256	0.0006738189029345373	768	25769803776	448.1766342333362	57499213.050413854	38244406121244.69	qk_init
50	3	256	256	128	0.000358254672959467	768	12884901888	421.47375986100144	30571065.425874516	35965760841472.125	qk_init
51	3	256	256	64	0.0002007051105022831	768	6442450944	376.1611839930762	17126836.096194826	32099087700742.5	qk_init
52	3	256	256	32	0.00012189697230142565	768	3221225472	309.6773881032524	10401874.969721656	26425803784810.87	qk_init
53	3	256	256	16	8.453561698040722e-05	768	1610612736	223.2711923587723	7213705.982328083	19052475081281.902	qk_init
54	3	256	256	8	6.407660705009276e-05	768	805306368	147.2797083750448	5467870.468274581	12567868448003.822	qk_init
55	3	256	256	4	5.036328747284576e-05	768	402653184	93.69110391262903	4297667.197682838	7994974200544.344	qk_init
56	3	256	256	2	4.5488761135057476e-05	768	201326592	51.865470527877875	3881707.616858238	4425853485045.578	qk_init
57	12	256	256	2048	0.020202365999999996	3072	824633720832	478.3437947812648	1723935231.9999998	40818670488001.266	qk_init
58	12	256	256	1024	0.010124155888157895	3072	412316860416	477.2583770318811	863927969.1228071	40726048173387.19	qk_init
59	12	256	256	512	0.005085633937062937	3072	206158430208	475.04777848703077	433974095.9627039	40537410430893.29	qk_init
60	12	256	256	256	0.0025654916853281853	3072	103079215104	470.84913933193053	218921957.14800516	40179126556324.74	qk_init
61	12	256	256	128	0.0013045765704467354	3072	51539607552	462.9699702434292	111323867.34478809	39506770794105.96	qk_init
62	12	256	256	64	0.0006742801519939804	3072	25769803776	447.87005387442576	57538572.970153	38218244597284.33	qk_init
63	12	256	256	32	0.00035831976790671853	3072	12884901888	421.3971919051604	30576620.194706645	35959227042573.69	qk_init
64	12	256	256	16	0.0002005369068918302	3072	6442450944	376.4766953382971	17112482.721436176	32126011335534.68	qk_init
65	12	256	256	8	0.00012179187250509165	3072	3221225472	309.94462293386505	10392906.453767821	26448607823689.82	qk_init
66	12	256	256	4	8.452507263643351e-05	3072	1610612736	223.2990450204527	7212806.198308992	19054851841745.297	qk_init
67	12	256	256	2	6.412381767545489e-05	3072	805306368	147.17127491946468	5471899.108305484	12558615459794.32	qk_init
68	3	256	256	2048	0.0016183739398395718	768	805306368	811597824.0	0.9922480620155039	1265467.7325087283	qk_ar
69	3	256	256	1024	0.0008322699728813558	768	402653184	405798912.0	0.9922480620155039	1230369.9921491416	qk_ar
70	3	256	256	512	0.00043886859397590365	768	201326592	202899456.0	0.9922480620155039	1166636.2255762408	qk_ar
71	3	256	256	256	0.00024185948322147648	768	100663296	101449728.0	0.9922480620155039	1058465.8355760013	qk_ar
72	3	256	256	128	0.00014308985100166944	768	50331648	50724864.0	0.9922480620155039	894542.82818777	qk_ar
73	3	256	256	64	9.382939365815932e-05	768	25165824	25362432.0	0.9922480620155039	682089.028872613	qk_ar
74	3	256	256	32	6.856070612244899e-05	768	12582912	12681216.0	0.9922480620155039	466739.6503012703	qk_ar
75	3	256	256	16	5.452260553129549e-05	768	6291456	6340608.0	0.9922480620155039	293456.26174846216	qk_ar
76	3	256	256	8	4.608557533261417e-05	768	3145728	3170304.0	0.9922480620155039	173590.1080166944	qk_ar
77	3	256	256	4	4.386146957766642e-05	768	1572864	1585152.0	0.9922480620155039	91196.21477609445	qk_ar
78	3	256	256	2	4.330941094420601e-05	768	786432	792576.0	0.9922480620155039	46179.33969539622	qk_ar
79	12	256	256	2048	0.006347041645299144	3072	3221225472	3246391296.0	0.9922480620155039	322670.011392918	qk_ar
80	12	256	256	1024	0.0031943104467592586	3072	1610612736	1623195648.0	0.9922480620155039	320569.96872013	qk_ar
81	12	256	256	512	0.0016183416350267381	3072	805306368	811597824.0	0.9922480620155039	316373.2483416833	qk_ar
82	12	256	256	256	0.0008325934893977947	3072	402653184	405798912.0	0.9922480620155039	307472.9784221131	qk_ar
83	12	256	256	128	0.0004389725746987952	3072	201326592	202899456.0	0.9922480620155039	291589.9702568624	qk_ar
84	12	256	256	64	0.00024191767449664432	3072	100663296	101449728.0	0.9922480620155039	264552.8076159138	qk_ar
85	12	256	256	32	0.0001431546143572621	3072	50331648	50724864.0	0.9922480620155039	223534.53392804778	qk_ar
86	12	256	256	16	9.404283597678917e-05	3072	25165824	25362432.0	0.9922480620155039	170135.23501087292	qk_ar
87	12	256	256	8	6.855550037091989e-05	3072	12582912	12681216.0	0.9922480620155039	116693.773026467	qk_ar
88	12	256	256	4	5.4802094978165945e-05	3072	6291456	6340608.0	0.9922480620155039	72989.91036006316	qk_ar
89	12	256	256	2	4.608510707869206e-05	3072	3145728	3170304.0	0.9922480620155039	43397.96795057727	qk_ar