-
Notifications
You must be signed in to change notification settings - Fork 98
/
Copy pathperf.yaml
686 lines (686 loc) · 17.5 KB
/
perf.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
aggregated:
supported_oses:
- Android
supported_devices:
- Samsung Galaxy S24
- Samsung Galaxy S24 Ultra
- Samsung Galaxy S24+
- Samsung Galaxy S23
- Samsung Galaxy S23 Ultra
- Samsung Galaxy S23+
- Samsung Galaxy S22 5G
- Samsung Galaxy S22 Ultra 5G
- Samsung Galaxy S22+ 5G
- Samsung Galaxy Tab S8
- Xiaomi 12
- Xiaomi 12 Pro
- Samsung Galaxy S21
- Samsung Galaxy S21 Ultra
- Samsung Galaxy S21+
- Snapdragon X Elite CRD
- QCS8550 (Proxy)
- SA8775 (Proxy)
- SA8650 (Proxy)
- SA8255 (Proxy)
- QCS8450 (Proxy)
- XR2 Gen 2 (Proxy)
- Google Pixel 5a 5G
- Google Pixel 4
- Google Pixel 4a
- Google Pixel 3
- Google Pixel 3a
- Google Pixel 3a XL
supported_chipsets:
- Snapdragon® 8 Gen 3
- Snapdragon® 8 Gen 2
- Snapdragon® 8 Gen 1
- Snapdragon® 888
- Snapdragon® X Elite
- Qcs8550
- Sa8775p
- Sa8650p
- Sa8255p
- Qcs8450
models:
- name: WhisperEncoder
performance_metrics:
- torchscript_onnx_tflite:
inference_time: 623350.0
throughput: 1.6042351808775166
estimated_peak_memory_range:
min: 102223872
max: 556720864
primary_compute_unit: GPU
precision: fp16
layer_info:
layers_on_npu: 0
layers_on_gpu: 911
layers_on_cpu: 0
total_layers: 911
job_id: jlpe67d1g
job_status: Passed
torchscript_onnx_qnn:
inference_time: 1890126.0
throughput: 0.5290652580833235
estimated_peak_memory_range:
min: 86016
max: 254815320
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 1329
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 1329
job_id: j2p0om46p
job_status: Passed
reference_device_info:
name: Samsung Galaxy S23
os: '13'
form_factor: Phone
os_name: Android
manufacturer: Samsung
chipset: Snapdragon® 8 Gen 2
timestamp: '2024-08-10T22:35:22Z'
- torchscript_onnx_tflite:
inference_time: 462336.0
throughput: 2.1629291251384273
estimated_peak_memory_range:
min: 111874048
max: 204887840
primary_compute_unit: GPU
precision: fp16
layer_info:
layers_on_npu: 0
layers_on_gpu: 911
layers_on_cpu: 0
total_layers: 911
job_id: jz5wylw6g
job_status: Passed
torchscript_onnx_qnn:
inference_time: 1404578.0
throughput: 0.7119576128915589
estimated_peak_memory_range:
min: 659456
max: 542879200
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 1329
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 1329
job_id: jogk62v25
job_status: Passed
torchscript_onnx:
inference_time: 1183349.0
throughput: 0.8450592344270371
estimated_peak_memory_range:
min: 55115776
max: 3115065952
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 884
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 884
job_id: jnp1ony85
job_status: Passed
reference_device_info:
name: Samsung Galaxy S24
os: '14'
form_factor: Phone
os_name: Android
manufacturer: Samsung
chipset: Snapdragon® 8 Gen 3
timestamp: '2024-08-10T22:35:42Z'
- torchscript_onnx_tflite:
inference_time: 610414.0
throughput: 1.638232412755933
estimated_peak_memory_range:
min: 0
max: 452809480
primary_compute_unit: GPU
precision: fp16
layer_info:
layers_on_npu: 0
layers_on_gpu: 911
layers_on_cpu: 0
total_layers: 911
job_id: jnp1on225
job_status: Passed
torchscript_onnx_qnn:
inference_time: 1279192.0
throughput: 0.7817434755689529
estimated_peak_memory_range:
min: 974848
max: 2210560
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 1329
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 1329
job_id: j1p3oznlp
job_status: Passed
reference_device_info:
name: QCS8550 (Proxy)
os: '12'
form_factor: Iot
os_name: Android
manufacturer: Qualcomm
chipset: Qcs8550
timestamp: '2024-08-10T22:35:29Z'
- torchscript_onnx_tflite:
inference_time: 829351.0
throughput: 1.2057620959039055
estimated_peak_memory_range:
min: 110411776
max: 210805296
primary_compute_unit: GPU
precision: fp16
layer_info:
layers_on_npu: 0
layers_on_gpu: 911
layers_on_cpu: 0
total_layers: 911
job_id: jz57oe2lg
job_status: Passed
reference_device_info:
name: QCS8450 (Proxy)
os: '13'
form_factor: Xr
os_name: Android
manufacturer: Qualcomm
chipset: Qcs8450
timestamp: '2024-08-10T22:35:13Z'
- torchscript_onnx_tflite:
inference_time: 613595.0
throughput: 1.62973948614314
estimated_peak_memory_range:
min: 2187264
max: 322938640
primary_compute_unit: GPU
precision: fp16
layer_info:
layers_on_npu: 0
layers_on_gpu: 911
layers_on_cpu: 0
total_layers: 911
job_id: j0px0l91p
job_status: Passed
torchscript_onnx_qnn:
inference_time: 1269373.0
throughput: 0.7877905075970577
estimated_peak_memory_range:
min: 749568
max: 1998944
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 1329
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 1329
job_id: j1pv2lqjg
job_status: Passed
reference_device_info:
name: SA8650 (Proxy)
os: '13'
form_factor: Auto
os_name: Android
manufacturer: Qualcomm
chipset: Sa8650p
timestamp: '2024-08-10T22:35:31Z'
- torchscript_onnx_tflite:
inference_time: 610375.0
throughput: 1.6383370878558263
estimated_peak_memory_range:
min: 16384
max: 444125352
primary_compute_unit: GPU
precision: fp16
layer_info:
layers_on_npu: 0
layers_on_gpu: 911
layers_on_cpu: 0
total_layers: 911
job_id: jegn1z0rp
job_status: Passed
torchscript_onnx_qnn:
inference_time: 1245092.0
throughput: 0.803153501909899
estimated_peak_memory_range:
min: 802816
max: 1929752
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 1329
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 1329
job_id: jlpe67o1g
job_status: Passed
reference_device_info:
name: SA8775 (Proxy)
os: '13'
form_factor: Auto
os_name: Android
manufacturer: Qualcomm
chipset: Sa8775p
timestamp: '2024-08-10T22:35:34Z'
- torchscript_onnx_tflite:
inference_time: 610977.0
throughput: 1.636722822626711
estimated_peak_memory_range:
min: 49229824
max: 495393512
primary_compute_unit: GPU
precision: fp16
layer_info:
layers_on_npu: 0
layers_on_gpu: 911
layers_on_cpu: 0
total_layers: 911
job_id: jep2orx4g
job_status: Passed
torchscript_onnx_qnn:
inference_time: 1294694.0
throughput: 0.7723832813004463
estimated_peak_memory_range:
min: 958464
max: 2351384
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 1329
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 1329
job_id: jz5wyl26g
job_status: Passed
reference_device_info:
name: SA8255 (Proxy)
os: '13'
form_factor: Auto
os_name: Android
manufacturer: Qualcomm
chipset: Sa8255p
timestamp: '2024-08-10T22:35:36Z'
- torchscript_onnx_qnn:
inference_time: 1087181.0
throughput: 0.9198100408303677
estimated_peak_memory_range:
min: 483328
max: 483328
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 1329
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 1329
job_id: j1glwy48p
job_status: Passed
torchscript_onnx:
inference_time: 1506275.0
throughput: 0.663889396026622
estimated_peak_memory_range:
min: 454066176
max: 454066176
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 884
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 884
job_id: jz57oe0vg
job_status: Passed
reference_device_info:
name: Snapdragon X Elite CRD
os: '11'
form_factor: Compute
os_name: Windows
manufacturer: Qualcomm
chipset: Snapdragon® X Elite
timestamp: '2024-08-10T22:35:45Z'
- name: WhisperDecoder
performance_metrics:
- torchscript_onnx_tflite:
inference_time: 54275.0
throughput: 18.424689083371717
estimated_peak_memory_range:
min: 128020480
max: 131109392
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2560
layers_on_gpu: 1
layers_on_cpu: 12
total_layers: 2573
job_id: jygzzl3kg
job_status: Passed
torchscript_onnx_qnn:
inference_time: 11949.0
throughput: 83.68901163277262
estimated_peak_memory_range:
min: 63619072
max: 133879384
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: j1p8je2x5
job_status: Passed
torchscript_onnx:
inference_time: 61557.0
throughput: 16.245106161768767
estimated_peak_memory_range:
min: 163840
max: 271544440
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2302
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2302
job_id: jmg9ozjwg
job_status: Passed
reference_device_info:
name: Samsung Galaxy S23
os: '13'
form_factor: Phone
os_name: Android
manufacturer: Samsung
chipset: Snapdragon® 8 Gen 2
timestamp: '2024-08-10T22:35:41Z'
- torchscript_onnx_tflite:
inference_time: 48761.0
throughput: 20.508193023112735
estimated_peak_memory_range:
min: 128675840
max: 759930960
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2560
layers_on_gpu: 1
layers_on_cpu: 12
total_layers: 2573
job_id: jmg9oz0lg
job_status: Passed
torchscript_onnx_qnn:
inference_time: 9326.0
throughput: 107.2271070126528
estimated_peak_memory_range:
min: 55336960
max: 147292400
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: jn5q4l04g
job_status: Passed
torchscript_onnx:
inference_time: 48370.0
throughput: 20.673971469919373
estimated_peak_memory_range:
min: 96161792
max: 1506791040
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2302
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2302
job_id: jvgd6derp
job_status: Passed
reference_device_info:
name: Samsung Galaxy S24
os: '14'
form_factor: Phone
os_name: Android
manufacturer: Samsung
chipset: Snapdragon® 8 Gen 3
timestamp: '2024-08-10T22:35:43Z'
- torchscript_onnx_tflite:
inference_time: 57007.0
throughput: 17.541705404599433
estimated_peak_memory_range:
min: 125976576
max: 132614968
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2560
layers_on_gpu: 1
layers_on_cpu: 12
total_layers: 2573
job_id: jvgd6dnep
job_status: Passed
torchscript_onnx_qnn:
inference_time: 12157.0
throughput: 82.25713580653121
estimated_peak_memory_range:
min: 63676416
max: 64868232
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: jwgodlzx5
job_status: Passed
reference_device_info:
name: QCS8550 (Proxy)
os: '12'
form_factor: Iot
os_name: Android
manufacturer: Qualcomm
chipset: Qcs8550
timestamp: '2024-08-10T22:35:29Z'
- torchscript_onnx_tflite:
inference_time: 64921.0
throughput: 15.403336362656152
estimated_peak_memory_range:
min: 128880640
max: 742622848
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2563
layers_on_gpu: 1
layers_on_cpu: 9
total_layers: 2573
job_id: jqp4eynvg
job_status: Passed
torchscript_onnx_qnn:
inference_time: 15534.0
throughput: 64.37491953135059
estimated_peak_memory_range:
min: 58114048
max: 156905632
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: jvgd6deep
job_status: Passed
reference_device_info:
name: QCS8450 (Proxy)
os: '13'
form_factor: Xr
os_name: Android
manufacturer: Qualcomm
chipset: Qcs8450
timestamp: '2024-08-10T22:35:39Z'
- torchscript_onnx_tflite:
inference_time: 56992.0
throughput: 17.54632229084784
estimated_peak_memory_range:
min: 125800448
max: 133750672
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2560
layers_on_gpu: 1
layers_on_cpu: 12
total_layers: 2573
job_id: jo5m90ewg
job_status: Passed
torchscript_onnx_qnn:
inference_time: 12230.0
throughput: 81.76614881439085
estimated_peak_memory_range:
min: 63721472
max: 65090320
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: j7gj3rdxp
job_status: Passed
reference_device_info:
name: SA8650 (Proxy)
os: '13'
form_factor: Auto
os_name: Android
manufacturer: Qualcomm
chipset: Sa8650p
timestamp: '2024-08-10T22:35:32Z'
- torchscript_onnx_tflite:
inference_time: 56508.0
throughput: 17.69660932965244
estimated_peak_memory_range:
min: 129130496
max: 138368720
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2560
layers_on_gpu: 1
layers_on_cpu: 12
total_layers: 2573
job_id: joprxl69p
job_status: Passed
torchscript_onnx_qnn:
inference_time: 11899.0
throughput: 84.04067568703252
estimated_peak_memory_range:
min: 67727360
max: 68977336
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: jygzzl2kg
job_status: Passed
reference_device_info:
name: SA8775 (Proxy)
os: '13'
form_factor: Auto
os_name: Android
manufacturer: Qualcomm
chipset: Sa8775p
timestamp: '2024-08-10T22:35:34Z'
- torchscript_onnx_tflite:
inference_time: 57362.0
throughput: 17.433143893169696
estimated_peak_memory_range:
min: 129142784
max: 132570824
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2560
layers_on_gpu: 1
layers_on_cpu: 12
total_layers: 2573
job_id: jqpy8oz7g
job_status: Passed
torchscript_onnx_qnn:
inference_time: 12427.0
throughput: 80.46994447573832
estimated_peak_memory_range:
min: 63705088
max: 73929240
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: jmg9ozjlg
job_status: Passed
reference_device_info:
name: SA8255 (Proxy)
os: '13'
form_factor: Auto
os_name: Android
manufacturer: Qualcomm
chipset: Sa8255p
timestamp: '2024-08-10T22:35:36Z'
- torchscript_onnx_qnn:
inference_time: 10684.0
throughput: 93.59790340696368
estimated_peak_memory_range:
min: 63696896
max: 63696896
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2255
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2255
job_id: jw56o8205
job_status: Passed
torchscript_onnx:
inference_time: 53768.0
throughput: 18.598422853742004
estimated_peak_memory_range:
min: 243802112
max: 243802112
primary_compute_unit: NPU
precision: fp16
layer_info:
layers_on_npu: 2302
layers_on_gpu: 0
layers_on_cpu: 0
total_layers: 2302
job_id: jqp4eyk8g
job_status: Passed
reference_device_info:
name: Snapdragon X Elite CRD
os: '11'
form_factor: Compute
os_name: Windows
manufacturer: Qualcomm
chipset: Snapdragon® X Elite
timestamp: '2024-08-10T22:35:45Z'