-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcdef_avx512.asm
868 lines (852 loc) · 31.2 KB
/
cdef_avx512.asm
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
; Copyright © 2020, VideoLAN and dav1d authors
; Copyright © 2020, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
; list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
; this list of conditions and the following disclaimer in the documentation
; and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%include "config.asm"
%include "ext/x86/x86inc.asm"
%if HAVE_AVX512ICL && ARCH_X86_64
%macro DUP4 1-*
%rep %0
times 4 db %1
%rotate 1
%endrep
%endmacro
%macro DIRS 16 ; cdef_directions[]
%rep 4 + 16 + 4 ; 6 7 0 1 2 3 4 5 6 7 0 1
; masking away unused bits allows us to use a single vpaddd {1to16}
; instruction instead of having to do vpbroadcastd + paddb
db %13 & 0x3f, -%13 & 0x3f
%rotate 1
%endrep
%endmacro
SECTION_RODATA 64
lut_perm_4x4: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79
db 16, 17, 0, 1, 2, 3, 4, 5, 18, 19, 8, 9, 10, 11, 12, 13
db 20, 21, 80, 81, 82, 83, 84, 85, 22, 23, 32, 33, 34, 35, 36, 37
db 98, 99,100,101,102,103,104,105, 50, 51, 52, 53, 54, 55, 56, 57
lut_perm_4x8a: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79
db 96, 97, 0, 1, 2, 3, 4, 5, 98, 99, 8, 9, 10, 11, 12, 13
lut_perm_4x8b:db 100,101, 16, 17, 18, 19, 20, 21,102,103, 24, 25, 26, 27, 28, 29
db 104,105, 32, 33, 34, 35, 36, 37,106,107, 40, 41, 42, 43, 44, 45
db 108,109, 48, 49, 50, 51, 52, 53,110,111, 56, 57, 58, 59, 60, 61
db 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95
pd_01234567: dd 0, 1, 2, 3, 4, 5, 6, 7
lut_perm_8x8a: db 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23
db -1, -1, 34, 35, 36, 37, 38, 39, -1, -1, 50, 51, 52, 53, 54, 55
db -1, -1, 66, 67, 68, 69, 70, 71, -1, -1, 82, 83, 84, 85, 86, 87
db 96, 97, 98, 99,100,101,102,103,112,113,114,115,116,117,118,119
lut_perm_8x8b: db 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23, 24, 25, 26, 27
db 36, 37, 38, 39, 40, 41, 42, 43, 52, 53, 54, 55, 56, 57, 58, 59
db 68, 69, 70, 71, 72, 73, 74, 75, 84, 85, 86, 87, 88, 89, 90, 91
db 100,101,102,103,104,105,106,107,116,117,118,119,120,121,122,123
edge_mask: dq 0x00003c3c3c3c0000, 0x00003f3f3f3f0000 ; 0000, 0001
dq 0x0000fcfcfcfc0000, 0x0000ffffffff0000 ; 0010, 0011
dq 0x00003c3c3c3c3c3c, 0x00003f3f3f3f3f3f ; 0100, 0101
dq 0x0000fcfcfcfcfcfc, 0x0000ffffffffffff ; 0110, 0111
dq 0x3c3c3c3c3c3c0000, 0x3f3f3f3f3f3f0000 ; 1000, 1001
dq 0xfcfcfcfcfcfc0000, 0xffffffffffff0000 ; 1010, 1011
dq 0x3c3c3c3c3c3c3c3c, 0x3f3f3f3f3f3f3f3f ; 1100, 1101
dq 0xfcfcfcfcfcfcfcfc, 0xffffffffffffffff ; 1110, 1111
px_idx: DUP4 18, 19, 20, 21, 26, 27, 28, 29, 34, 35, 36, 37, 42, 43, 44, 45
cdef_dirs: DIRS -7,-14, 1, -6, 1, 2, 1, 10, 9, 18, 8, 17, 8, 16, 8, 15
gf_shr: dq 0x0102040810204080, 0x0102040810204080 ; >> 0, >> 0
dq 0x0204081020408000, 0x0408102040800000 ; >> 1, >> 2
dq 0x0810204080000000, 0x1020408000000000 ; >> 3, >> 4
dq 0x2040800000000000, 0x4080000000000000 ; >> 5, >> 6
times 16 db 0 ; realign (introduced by cdef_dirs)
end_perm_w8clip:db 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30
db 32, 36, 40, 44, 34, 38, 42, 46, 48, 52, 56, 60, 50, 54, 58, 62
db 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31
db 33, 37, 41, 45, 35, 39, 43, 47, 49, 53, 57, 61, 51, 55, 59, 63
end_perm: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63
pri_tap: db 64, 64, 32, 32, 48, 48, 48, 48 ; left-shifted by 4
sec_tap: db 32, 32, 16, 16
pd_268435568: dd 268435568
SECTION .text
%if WIN64
DECLARE_REG_TMP 5, 6
%else
DECLARE_REG_TMP 8, 5
%endif
; lut:
; t0 t1 t2 t3 t4 t5 t6 t7
; T0 T1 T2 T3 T4 T5 T6 T7
; L0 L1 00 01 02 03 04 05
; L2 L3 10 11 12 13 14 15
; L4 L5 20 21 22 23 24 25
; L6 L7 30 31 32 33 34 35
; 4e 4f 40 41 42 43 44 45
; 5e 5f 50 51 52 53 54 55
INIT_ZMM avx512icl
cglobal cdef_filter_4x4_8bpc, 4, 8, 13, dst, stride, left, top, \
pri, sec, dir, damping, edge
%define base r7-edge_mask
movq xmm0, [dstq+strideq*0]
movhps xmm0, [dstq+strideq*1]
lea r7, [edge_mask]
movq xmm1, [topq+strideq*0-2]
movhps xmm1, [topq+strideq*1-2]
mov r6d, edgem
vinserti32x4 ym0, ymm0, [leftq], 1
lea r2, [strideq*3]
vinserti32x4 ym1, ymm1, [dstq+strideq*2], 1
mova m5, [base+lut_perm_4x4]
vinserti32x4 m0, [dstq+r2], 2
test r6b, 0x08 ; avoid buffer overread
jz .main
lea r3, [dstq+strideq*4-4]
vinserti32x4 m1, [r3+strideq*0], 2
vinserti32x4 m0, [r3+strideq*1], 3
.main:
movifnidn prid, prim
mov t0d, dirm
mova m3, [base+px_idx]
mov r3d, dampingm
vpermi2b m5, m0, m1 ; lut
vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4)
pxor m7, m7
lea r3, [r7+r3*8] ; gf_shr + (damping - 30) * 8
vpermb m6, m3, m5 ; px
cmp r6d, 0x0f
jne .mask_edges ; mask edges only if required
test prid, prid
jz .sec_only
vpaddd m1, m3, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir
vpermb m1, m1, m5 ; k0p0 k0p1 k1p0 k1p1
%macro CDEF_FILTER_4x4_PRI 0
vpcmpub k1, m6, m1, 6 ; px > pN
psubb m2, m1, m6
lzcnt r6d, prid
vpsubb m2{k1}, m6, m1 ; abs(diff)
vpbroadcastb m4, prid
and prid, 1
vgf2p8affineqb m9, m2, [r3+r6*8] {1to8}, 0 ; abs(diff) >> shift
movifnidn t1d, secm
vpbroadcastd m10, [base+pri_tap+priq*4]
vpsubb m10{k1}, m7, m10 ; apply_sign(pri_tap)
psubusb m4, m9 ; imax(0, pri_strength - (abs(diff) >> shift)))
pminub m2, m4
vpdpbusd m0, m2, m10 ; sum
%endmacro
CDEF_FILTER_4x4_PRI
test t1d, t1d ; sec
jz .end_no_clip
call .sec
.end_clip:
pminub m4, m6, m1
pmaxub m1, m6
pminub m5, m2, m3
pmaxub m2, m3
pminub m4, m5
pmaxub m2, m1
psrldq m1, m4, 2
psrldq m3, m2, 2
pminub m1, m4
vpcmpw k1, m0, m7, 1
vpshldd m6, m0, 8
pmaxub m2, m3
pslldq m3, m1, 1
psubw m7, m0
paddusw m0, m6 ; clip >0xff
vpsubusw m0{k1}, m6, m7 ; clip <0x00
pslldq m4, m2, 1
pminub m1, m3
pmaxub m2, m4
pmaxub m0, m1
pminub m0, m2
jmp .end
.sec_only:
movifnidn t1d, secm
call .sec
.end_no_clip:
vpshldd m6, m0, 8 ; (px << 8) + ((sum > -8) << 4)
paddw m0, m6 ; (px << 8) + ((sum + (sum > -8) + 7) << 4)
.end:
mova xm1, [base+end_perm]
vpermb m0, m1, m0 ; output in bits 8-15 of each dword
movd [dstq+strideq*0], xm0
pextrd [dstq+strideq*1], xm0, 1
pextrd [dstq+strideq*2], xm0, 2
pextrd [dstq+r2 ], xm0, 3
RET
.mask_edges_sec_only:
movifnidn t1d, secm
call .mask_edges_sec
jmp .end_no_clip
ALIGN function_align
.mask_edges:
vpbroadcastq m8, [base+edge_mask+r6*8]
test prid, prid
jz .mask_edges_sec_only
vpaddd m2, m3, [base+cdef_dirs+(t0+2)*4] {1to16}
vpshufbitqmb k1, m8, m2 ; index in-range
mova m1, m6
vpermb m1{k1}, m2, m5
CDEF_FILTER_4x4_PRI
test t1d, t1d
jz .end_no_clip
call .mask_edges_sec
jmp .end_clip
.mask_edges_sec:
vpaddd m4, m3, [base+cdef_dirs+(t0+4)*4] {1to16}
vpaddd m9, m3, [base+cdef_dirs+(t0+0)*4] {1to16}
vpshufbitqmb k1, m8, m4
mova m2, m6
vpermb m2{k1}, m4, m5
vpshufbitqmb k1, m8, m9
mova m3, m6
vpermb m3{k1}, m9, m5
jmp .sec_main
ALIGN function_align
.sec:
vpaddd m2, m3, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2
vpaddd m3, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2
vpermb m2, m2, m5 ; k0s0 k0s1 k1s0 k1s1
vpermb m3, m3, m5 ; k0s2 k0s3 k1s2 k1s3
.sec_main:
vpbroadcastd m8, [base+sec_tap]
vpcmpub k1, m6, m2, 6
psubb m4, m2, m6
vpbroadcastb m12, t1d
lzcnt t1d, t1d
vpsubb m4{k1}, m6, m2
vpcmpub k2, m6, m3, 6
vpbroadcastq m11, [r3+t1*8]
gf2p8affineqb m10, m4, m11, 0
psubb m5, m3, m6
mova m9, m8
vpsubb m8{k1}, m7, m8
psubusb m10, m12, m10
vpsubb m5{k2}, m6, m3
pminub m4, m10
vpdpbusd m0, m4, m8
gf2p8affineqb m11, m5, m11, 0
vpsubb m9{k2}, m7, m9
psubusb m12, m11
pminub m5, m12
vpdpbusd m0, m5, m9
ret
DECLARE_REG_TMP 2, 7
; lut top lut bottom
; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25
; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35
; L0 L1 00 01 02 03 04 05 L8 L9 40 41 42 43 44 45
; L2 L3 10 11 12 13 14 15 La Lb 50 51 52 53 54 55
; L4 L5 20 21 22 23 24 25 Lc Ld 60 61 62 63 64 65
; L6 L7 30 31 32 33 34 35 Le Lf 70 71 72 73 74 75
; L8 L9 40 41 42 43 44 45 8e 8f 80 81 82 83 84 85
; La Lb 50 51 52 53 54 55 9e 9f 90 91 92 93 94 95
cglobal cdef_filter_4x8_8bpc, 4, 9, 22, dst, stride, left, top, pri, sec, dir, damping, edge
%define base r8-edge_mask
vpbroadcastd ym21, strided
mov r6d, edgem
lea r8, [edge_mask]
movq xm1, [topq+strideq*0-2]
pmulld ym21, [base+pd_01234567]
kxnorb k1, k1, k1
movq xm2, [topq+strideq*1-2]
vpgatherdq m0{k1}, [dstq+ym21] ; +0+1 +2+3 +4+5 +6+7
mova m14, [base+lut_perm_4x8a]
movu m15, [base+lut_perm_4x8b]
test r6b, 0x08 ; avoid buffer overread
jz .main
lea r7, [dstq+strideq*8-2]
vinserti32x4 ym1, [r7+strideq*0], 1
vinserti32x4 ym2, [r7+strideq*1], 1
.main:
punpcklqdq ym1, ym2
vinserti32x4 m1, [leftq], 2 ; -2-1 +8+9 left ____
movifnidn prid, prim
mov t0d, dirm
mova m16, [base+px_idx]
mov r3d, dampingm
vpermi2b m14, m0, m1 ; lut top
vpermi2b m15, m0, m1 ; lut bottom
vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4)
pxor m20, m20
lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8
vpermb m2, m16, m14 ; pxt
vpermb m3, m16, m15 ; pxb
mova m1, m0
cmp r6b, 0x0f
jne .mask_edges ; mask edges only if required
test prid, prid
jz .sec_only
vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir
vpermb m4, m6, m14 ; pNt k0p0 k0p1 k1p0 k1p1
vpermb m5, m6, m15 ; pNb
%macro CDEF_FILTER_4x8_PRI 0
vpcmpub k1, m2, m4, 6 ; pxt > pNt
vpcmpub k2, m3, m5, 6 ; pxb > pNb
psubb m6, m4, m2
psubb m7, m5, m3
lzcnt r6d, prid
vpsubb m6{k1}, m2, m4 ; abs(diff_top)
vpsubb m7{k2}, m3, m5 ; abs(diff_bottom)
vpbroadcastb m13, prid
vpbroadcastq m9, [r3+r6*8]
and prid, 1
vpbroadcastd m11, [base+pri_tap+priq*4]
vgf2p8affineqb m8, m6, m9, 0 ; abs(dt) >> shift
vgf2p8affineqb m9, m7, m9, 0 ; abs(db) >> shift
mova m10, m11
movifnidn t1d, secm
vpsubb m10{k1}, m20, m11 ; apply_sign(pri_tap_top)
vpsubb m11{k2}, m20, m11 ; apply_sign(pri_tap_bottom)
psubusb m12, m13, m8 ; imax(0, pri_strength - (abs(dt) >> shift)))
psubusb m13, m13, m9 ; imax(0, pri_strength - (abs(db) >> shift)))
pminub m6, m12
pminub m7, m13
vpdpbusd m0, m6, m10 ; sum top
vpdpbusd m1, m7, m11 ; sum bottom
%endmacro
CDEF_FILTER_4x8_PRI
test t1d, t1d ; sec
jz .end_no_clip
call .sec
.end_clip:
pminub m10, m4, m2
pminub m12, m6, m8
pminub m11, m5, m3
pminub m13, m7, m9
pmaxub m4, m2
pmaxub m6, m8
pmaxub m5, m3
pmaxub m7, m9
pminub m10, m12
pminub m11, m13
pmaxub m4, m6
pmaxub m5, m7
mov r2d, 0xAAAAAAAA
kmovd k1, r2d
kxnorb k2, k2, k2 ; hw lw
vpshrdd m12, m0, m1, 16 ; m1lw m0hw
vpshrdd m6, m10, m11, 16 ; m11lw m10hw
vpshrdd m8, m4, m5, 16 ; m5lw m4hw
vpblendmw m7{k1}, m10, m11 ; m11hw m10lw
vpblendmw m9{k1}, m4, m5 ; m5hw m4lw
vpblendmw m4{k1}, m0, m12 ; m1lw m0lw
vpblendmw m5{k1}, m12, m1 ; m1hw m0hw
vpshrdd m2, m3, 16
pminub m6, m7
pmaxub m8, m9
mova ym14, [base+end_perm]
vpcmpw k1, m4, m20, 1
vpshldw m2, m5, 8
pslldq m7, m6, 1
pslldq m9, m8, 1
psubw m5, m20, m4
paddusw m0, m4, m2 ; clip >0xff
pminub m6, m7
pmaxub m8, m9
psubusw m0{k1}, m2, m5 ; clip <0x00
pmaxub m0, m6
pminub m0, m8
vpermb m0, m14, m0
vpscatterdd [dstq+ym21]{k2}, ym0
RET
.sec_only:
movifnidn t1d, secm
call .sec
.end_no_clip:
mova ym4, [base+end_perm]
kxnorb k1, k1, k1
vpshldd m2, m0, 8 ; (px << 8) + ((sum > -8) << 4)
vpshldd m3, m1, 8
paddw m0, m2 ; (px << 8) + ((sum + (sum > -8) + 7) << 4)
paddw m1, m3
pslld m0, 16
vpshrdd m0, m1, 16
vpermb m0, m4, m0 ; output in bits 8-15 of each word
vpscatterdd [dstq+ym21]{k1}, ym0
RET
.mask_edges_sec_only:
movifnidn t1d, secm
call .mask_edges_sec
jmp .end_no_clip
ALIGN function_align
.mask_edges:
mov t1d, r6d
or r6d, 8 ; top 4x4 has bottom
or t1d, 4 ; bottom 4x4 has top
vpbroadcastq m17, [base+edge_mask+r6*8]
vpbroadcastq m18, [base+edge_mask+t1*8]
test prid, prid
jz .mask_edges_sec_only
vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16}
vpshufbitqmb k1, m17, m6 ; index in-range
vpshufbitqmb k2, m18, m6
mova m4, m2
mova m5, m3
vpermb m4{k1}, m6, m14
vpermb m5{k2}, m6, m15
CDEF_FILTER_4x8_PRI
test t1d, t1d
jz .end_no_clip
call .mask_edges_sec
jmp .end_clip
.mask_edges_sec:
vpaddd m10, m16, [base+cdef_dirs+(t0+4)*4] {1to16}
vpaddd m11, m16, [base+cdef_dirs+(t0+0)*4] {1to16}
vpshufbitqmb k1, m17, m10
vpshufbitqmb k2, m18, m10
vpshufbitqmb k3, m17, m11
vpshufbitqmb k4, m18, m11
mova m6, m2
mova m7, m3
mova m8, m2
mova m9, m3
vpermb m6{k1}, m10, m14
vpermb m7{k2}, m10, m15
vpermb m8{k3}, m11, m14
vpermb m9{k4}, m11, m15
jmp .sec_main
ALIGN function_align
.sec:
vpaddd m8, m16, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2
vpaddd m9, m16, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2
vpermb m6, m8, m14 ; pNt k0s0 k0s1 k1s0 k1s1
vpermb m7, m8, m15 ; pNb
vpermb m8, m9, m14 ; pNt k0s2 k0s3 k1s2 k1s3
vpermb m9, m9, m15 ; pNb
.sec_main:
vpbroadcastb m18, t1d
lzcnt t1d, t1d
vpcmpub k1, m2, m6, 6
vpcmpub k2, m3, m7, 6
vpcmpub k3, m2, m8, 6
vpcmpub k4, m3, m9, 6
vpbroadcastq m17, [r3+t1*8]
psubb m10, m6, m2
psubb m11, m7, m3
psubb m12, m8, m2
psubb m13, m9, m3
vpsubb m10{k1}, m2, m6 ; abs(dt0)
vpsubb m11{k2}, m3, m7 ; abs(db0)
vpsubb m12{k3}, m2, m8 ; abs(dt1)
vpsubb m13{k4}, m3, m9 ; abs(db1)
vpbroadcastd m19, [base+sec_tap]
gf2p8affineqb m14, m10, m17, 0 ; abs(dt0) >> shift
gf2p8affineqb m15, m11, m17, 0 ; abs(db0) >> shift
gf2p8affineqb m16, m12, m17, 0 ; abs(dt1) >> shift
gf2p8affineqb m17, m13, m17, 0 ; abs(db1) >> shift
psubusb m14, m18, m14 ; imax(0, sec_strength - (abs(dt0) >> shift)))
psubusb m15, m18, m15 ; imax(0, sec_strength - (abs(db0) >> shift)))
psubusb m16, m18, m16 ; imax(0, sec_strength - (abs(dt1) >> shift)))
psubusb m17, m18, m17 ; imax(0, sec_strength - (abs(db1) >> shift)))
pminub m10, m14
pminub m11, m15
pminub m12, m16
pminub m13, m17
mova m14, m19
mova m15, m19
mova m16, m19
vpsubb m14{k1}, m20, m19 ; apply_sign(sec_tap_top_0)
vpsubb m15{k2}, m20, m19 ; apply_sign(sec_tap_bottom_0)
vpsubb m16{k3}, m20, m19 ; apply_sign(sec_tap_top_1)
vpsubb m19{k4}, m20, m19 ; apply_sign(sec_tap_bottom_1)
vpdpbusd m0, m10, m14
vpdpbusd m1, m11, m15
vpdpbusd m0, m12, m16
vpdpbusd m1, m13, m19
ret
; lut tl lut tr
; t0 t1 t2 t3 t4 t5 t6 t7 t6 t7 t8 t9 ta tb tc td
; T0 T1 T2 T3 T4 T5 T6 T7 T6 T7 T8 T9 TA TB TC TD
; L0 L1 00 01 02 03 04 05 04 05 06 07 08 09 0a 0b
; L2 L3 10 11 12 13 14 15 14 15 16 17 18 19 1a 1b
; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b
; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b
; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b
; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b
; lut bl lut br
; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b
; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b
; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b
; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b
; Lc Ld 60 61 62 63 64 65 64 65 66 67 68 69 6a 6b
; Le Lf 70 71 72 73 74 75 74 75 76 77 78 79 7a 7b
; 8e 8f 80 81 82 83 84 85 84 85 86 87 88 89 8a 8b
; 9e 9f 90 91 92 93 94 95 94 95 96 97 98 99 9a 9b
cglobal cdef_filter_8x8_8bpc, 4, 11, 32, 4*64, dst, stride, left, top, \
pri, sec, dir, damping, edge
%define base r8-edge_mask
mov r6d, edgem
lea r10, [dstq+strideq*4-2]
movu xmm0, [topq+strideq*0-2]
movu xmm1, [dstq+strideq*2-2]
movu xmm2, [r10 +strideq*2 ]
lea r8, [edge_mask]
lea r9, [strideq*3]
pmovzxwq m10, [leftq-4]
vinserti32x4 ym0, ymm0, [topq+strideq*1-2], 1
vinserti32x4 ym1, ymm1, [dstq+r9 -2], 1
vinserti32x4 ym2, ymm2, [r10 +r9 ], 1
lea r7, [r10 +strideq*4 ]
pmovzxwq m11, [leftq+4]
vinserti32x4 m0, [dstq+strideq*0-2], 2
vinserti32x4 m1, [r10 +strideq*0 ], 2
mova m12, [base+lut_perm_8x8a]
movu m13, [base+lut_perm_8x8b]
vinserti32x4 m0, [dstq+strideq*1-2], 3
vinserti32x4 m1, [r10 +strideq*1 ], 3
test r6b, 0x08 ; avoid buffer overread
jz .main
vinserti32x4 m2, [r7 +strideq*0], 2
vinserti32x4 m2, [r7 +strideq*1], 3
.main:
mov t1d, 0x11111100
mova m14, m12
mova m15, m13
kmovd k1, t1d
kshiftrd k2, k1, 8
movifnidn prid, prim
mov t0d, dirm
mova m30, [base+px_idx]
mov r3d, dampingm
vpermi2b m12, m0, m1 ; lut tl
vpermi2b m14, m1, m2 ; lut bl
vpermi2b m13, m0, m1 ; lut tr
vpermi2b m15, m1, m2 ; lut br
vpblendmw m12{k1}, m12, m10
vpblendmw m14{k2}, m14, m11
vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4)
pxor m31, m31
lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8
vpermb m4, m30, m12 ; pxtl
vpermb m5, m30, m13 ; pxtr
vpermb m6, m30, m14 ; pxbl
vpermb m7, m30, m15 ; pxbr
mova m1, m0
mova m2, m0
mova m3, m0
cmp r6b, 0x0f
jne .mask_edges ; mask edges only if required
test prid, prid
jz .sec_only
vpaddd m11, m30, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir
vpermb m8, m11, m12 ; pNtl k0p0 k0p1 k1p0 k1p1
vpermb m9, m11, m13 ; pNtr
vpermb m10, m11, m14 ; pNbl
vpermb m11, m11, m15 ; pNbr
%macro CDEF_FILTER_8x8_PRI 0
vpcmpub k1, m4, m8, 6 ; pxtl > pNtl
vpcmpub k2, m5, m9, 6 ; pxtr > pNtr
vpcmpub k3, m6, m10, 6 ; pxbl > pNbl
vpcmpub k4, m7, m11, 6 ; pxbr > pNbr
psubb m16, m8, m4
psubb m17, m9, m5
psubb m18, m10, m6
psubb m19, m11, m7
lzcnt r6d, prid
vpsubb m16{k1}, m4, m8 ; abs(diff_tl)
vpsubb m17{k2}, m5, m9 ; abs(diff_tr)
vpsubb m18{k3}, m6, m10 ; abs(diff_bl)
vpsubb m19{k4}, m7, m11 ; abs(diff_br)
vpbroadcastq m28, [r3+r6*8]
vpbroadcastb m29, prid
and prid, 1
vpbroadcastd m27, [base+pri_tap+priq*4]
vgf2p8affineqb m20, m16, m28, 0 ; abs(dtl) >> shift
vgf2p8affineqb m21, m17, m28, 0 ; abs(dtr) >> shift
vgf2p8affineqb m22, m18, m28, 0 ; abs(dbl) >> shift
vgf2p8affineqb m23, m19, m28, 0 ; abs(dbl) >> shift
mova m24, m27
mova m25, m27
mova m26, m27
movifnidn t1d, secm
vpsubb m24{k1}, m31, m27 ; apply_sign(pri_tap_tl)
vpsubb m25{k2}, m31, m27 ; apply_sign(pri_tap_tr)
vpsubb m26{k3}, m31, m27 ; apply_sign(pri_tap_tl)
vpsubb m27{k4}, m31, m27 ; apply_sign(pri_tap_tr)
psubusb m20, m29, m20 ; imax(0, pri_strength - (abs(dtl) >> shift)))
psubusb m21, m29, m21 ; imax(0, pri_strength - (abs(dtr) >> shift)))
psubusb m22, m29, m22 ; imax(0, pri_strength - (abs(dbl) >> shift)))
psubusb m23, m29, m23 ; imax(0, pri_strength - (abs(dbr) >> shift)))
pminub m16, m20
pminub m17, m21
pminub m18, m22
pminub m19, m23
vpdpbusd m0, m16, m24 ; sum tl
vpdpbusd m1, m17, m25 ; sum tr
vpdpbusd m2, m18, m26 ; sum bl
vpdpbusd m3, m19, m27 ; sum br
%endmacro
CDEF_FILTER_8x8_PRI
test t1d, t1d ; sec
jz .end_no_clip
call .sec
.end_clip:
pminub m20, m8, m4
pminub m24, m12, m16
pminub m21, m9, m5
pminub m25, m13, m17
pminub m22, m10, m6
pminub m26, m14, m18
pminub m23, m11, m7
pminub m27, m15, m19
pmaxub m8, m4
pmaxub m12, m16
pmaxub m9, m5
pmaxub m13, m17
pmaxub m10, m6
pmaxub m14, m18
pmaxub m11, m7
pmaxub m15, m19
pminub m20, m24
pminub m21, m25
pminub m22, m26
pminub m23, m27
pmaxub m8, m12
pmaxub m9, m13
pmaxub m10, m14
pmaxub m11, m15
mov r2d, 0xAAAAAAAA
kmovd k1, r2d
vpshrdd m24, m0, m1, 16
vpshrdd m25, m2, m3, 16
vpshrdd m12, m20, m21, 16
vpshrdd m14, m22, m23, 16
vpshrdd m16, m8, m9, 16
vpshrdd m18, m10, m11, 16
vpblendmw m13{k1}, m20, m21
vpblendmw m15{k1}, m22, m23
vpblendmw m17{k1}, m8, m9
vpblendmw m19{k1}, m10, m11
vpblendmw m20{k1}, m0, m24
vpblendmw m21{k1}, m24, m1
vpblendmw m22{k1}, m2, m25
vpblendmw m23{k1}, m25, m3
vpshrdd m4, m5, 16
vpshrdd m6, m7, 16
pminub m12, m13
pminub m14, m15
pmaxub m16, m17
pmaxub m18, m19
mova m8, [base+end_perm_w8clip]
vpcmpw k2, m20, m31, 1
vpcmpw k3, m22, m31, 1
vpshldw m4, m21, 8
vpshldw m6, m23, 8
kunpckdq k1, k1, k1
kxnorb k4, k4, k4
vpshrdw m11, m12, m14, 8
vpshrdw m15, m16, m18, 8
vpblendmb m13{k1}, m12, m14
vpblendmb m17{k1}, m16, m18
psubw m21, m31, m20
psubw m23, m31, m22
paddusw m0, m20, m4 ; clip >0xff
paddusw m1, m22, m6
pminub m11, m13
pmaxub m15, m17
psubusw m0{k2}, m4, m21 ; clip <0x00
psubusw m1{k3}, m6, m23
psrlw m0, 8
vmovdqu8 m0{k1}, m1
pmaxub m0, m11
pminub m0, m15
vpermb m0, m8, m0
add r10, 2
vextracti32x4 xm1, m0, 1
vextracti32x4 xm2, m0, 2
vextracti32x4 xm3, m0, 3
movq [dstq+strideq*0], xm0
movq [dstq+strideq*2], xm1
movq [r10 +strideq*0], xm2
movq [r10 +strideq*2], xm3
movhps [dstq+strideq*1], xm0
movhps [dstq+r9 ], xm1
movhps [r10 +strideq*1], xm2
movhps [r10 +r9 ], xm3
RET
.sec_only:
movifnidn t1d, secm
call .sec
.end_no_clip:
mova xm8, [base+end_perm]
kxnorb k1, k1, k1
vpshldd m4, m0, 8 ; (px << 8) + ((sum > -8) << 4)
vpshldd m5, m1, 8
vpshldd m6, m2, 8
vpshldd m7, m3, 8
paddw m0, m4 ; (px << 8) + ((sum + (sum > -8) + 7) << 4)
paddw m1, m5
paddw m2, m6
paddw m3, m7
vpermb m0, m8, m0
vpermb m1, m8, m1
vpermb m2, m8, m2
vpermb m3, m8, m3
add r10, 2
punpckldq m4, m0, m1
punpckhdq m0, m1
punpckldq m5, m2, m3
punpckhdq m2, m3
movq [dstq+strideq*0], xm4
movq [dstq+strideq*2], xm0
movq [r10 +strideq*0], xm5
movq [r10 +strideq*2], xm2
movhps [dstq+strideq*1], xm4
movhps [dstq+r9 ], xm0
movhps [r10 +strideq*1], xm5
movhps [r10 +r9 ], xm2
RET
.mask_edges_sec_only:
movifnidn t1d, secm
call .mask_edges_sec
jmp .end_no_clip
ALIGN function_align
.mask_edges:
mov t0d, r6d
mov t1d, r6d
or t0d, 0xA ; top-left 4x4 has bottom and right
or t1d, 0x9 ; top-right 4x4 has bottom and left
vpbroadcastq m26, [base+edge_mask+t0*8]
vpbroadcastq m27, [base+edge_mask+t1*8]
mov t1d, r6d
or r6d, 0x6 ; bottom-left 4x4 has top and right
or t1d, 0x5 ; bottom-right 4x4 has top and left
vpbroadcastq m28, [base+edge_mask+r6*8]
vpbroadcastq m29, [base+edge_mask+t1*8]
mov t0d, dirm
test prid, prid
jz .mask_edges_sec_only
vpaddd m20, m30, [base+cdef_dirs+(t0+2)*4] {1to16}
vpshufbitqmb k1, m26, m20 ; index in-range
vpshufbitqmb k2, m27, m20
vpshufbitqmb k3, m28, m20
vpshufbitqmb k4, m29, m20
mova m8, m4
mova m9, m5
mova m10, m6
mova m11, m7
vpermb m8{k1}, m20, m12
vpermb m9{k2}, m20, m13
vpermb m10{k3}, m20, m14
vpermb m11{k4}, m20, m15
mova [rsp+0x00], m26
mova [rsp+0x40], m27
mova [rsp+0x80], m28
mova [rsp+0xC0], m29
CDEF_FILTER_8x8_PRI
test t1d, t1d
jz .end_no_clip
mova m26, [rsp+0x00]
mova m27, [rsp+0x40]
mova m28, [rsp+0x80]
mova m29, [rsp+0xC0]
call .mask_edges_sec
jmp .end_clip
.mask_edges_sec:
vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16}
vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16}
vpshufbitqmb k1, m26, m20
vpshufbitqmb k2, m27, m20
vpshufbitqmb k3, m28, m20
vpshufbitqmb k4, m29, m20
mova m16, m4
mova m17, m5
mova m18, m6
mova m19, m7
vpermb m16{k1}, m20, m12
vpermb m17{k2}, m20, m13
vpermb m18{k3}, m20, m14
vpermb m19{k4}, m20, m15
vpshufbitqmb k1, m26, m21
vpshufbitqmb k2, m27, m21
vpshufbitqmb k3, m28, m21
vpshufbitqmb k4, m29, m21
vpermb m12, m21, m12
vpermb m13, m21, m13
vpermb m14, m21, m14
vpermb m15, m21, m15
vpblendmb m12{k1}, m4, m12
vpblendmb m13{k2}, m5, m13
vpblendmb m14{k3}, m6, m14
vpblendmb m15{k4}, m7, m15
jmp .sec_main
ALIGN function_align
.sec:
vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2
vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2
vpermb m16, m20, m12 ; pNtl k0s0 k0s1 k1s0 k1s1
vpermb m17, m20, m13 ; pNtr
vpermb m18, m20, m14 ; pNbl
vpermb m19, m20, m15 ; pNbr
vpermb m12, m21, m12 ; pNtl k0s2 k0s3 k1s2 k1s3
vpermb m13, m21, m13 ; pNtr
vpermb m14, m21, m14 ; pNbl
vpermb m15, m21, m15 ; pNbr
.sec_main:
%macro CDEF_FILTER_8x8_SEC 4-5 0 ; load constants
vpcmpub k1, m4, %1, 6
vpcmpub k2, m5, %2, 6
vpcmpub k3, m6, %3, 6
vpcmpub k4, m7, %4, 6
psubb m20, %1, m4
psubb m21, %2, m5
psubb m22, %3, m6
psubb m23, %4, m7
%if %5
vpbroadcastb m28, t1d
lzcnt t1d, t1d
vpbroadcastq m29, [r3+t1*8]
%endif
vpsubb m20{k1}, m4, %1
vpsubb m21{k2}, m5, %2
vpsubb m22{k3}, m6, %3
vpsubb m23{k4}, m7, %4
gf2p8affineqb m24, m20, m29, 0
gf2p8affineqb m25, m21, m29, 0
gf2p8affineqb m26, m22, m29, 0
gf2p8affineqb m27, m23, m29, 0
%if %5
vpbroadcastd m30, [base+sec_tap]
%endif
psubusb m24, m28, m24
psubusb m25, m28, m25
psubusb m26, m28, m26
psubusb m27, m28, m27
pminub m20, m24
pminub m21, m25
pminub m22, m26
pminub m23, m27
mova m24, m30
mova m25, m30
mova m26, m30
mova m27, m30
vpsubb m24{k1}, m31, m30
vpsubb m25{k2}, m31, m30
vpsubb m26{k3}, m31, m30
vpsubb m27{k4}, m31, m30
vpdpbusd m0, m20, m24
vpdpbusd m1, m21, m25
vpdpbusd m2, m22, m26
vpdpbusd m3, m23, m27
%endmacro
CDEF_FILTER_8x8_SEC m16, m17, m18, m19, 1
CDEF_FILTER_8x8_SEC m12, m13, m14, m15
ret
%endif ; HAVE_AVX512ICL && ARCH_X86_64