#include "libavutil/arm/asm.S"
#include "neon.S"
function ff_vp8_luma_dc_wht_neon, export=1
vld1.16 {q0-q1}, [r1,:128]
vmov.i16 q15, #0
vadd.i16 d4, d0, d3
vadd.i16 d6, d1, d2
vst1.16 {q15}, [r1,:128]!
vsub.i16 d7, d1, d2
vsub.i16 d5, d0, d3
vst1.16 {q15}, [r1,:128]
vadd.i16 q0, q2, q3
vsub.i16 q1, q2, q3
vmov.i16 q8, #3
vtrn.32 d0, d2
vtrn.32 d1, d3
vtrn.16 d0, d1
vtrn.16 d2, d3
vadd.i16 d0, d0, d16
vadd.i16 d4, d0, d3
vadd.i16 d6, d1, d2
vsub.i16 d7, d1, d2
vsub.i16 d5, d0, d3
vadd.i16 q0, q2, q3
vsub.i16 q1, q2, q3
vshr.s16 q0, q0, #3
vshr.s16 q1, q1, #3
mov r3, #32
vst1.16 {d0[0]}, [r0,:16], r3
vst1.16 {d1[0]}, [r0,:16], r3
vst1.16 {d2[0]}, [r0,:16], r3
vst1.16 {d3[0]}, [r0,:16], r3
vst1.16 {d0[1]}, [r0,:16], r3
vst1.16 {d1[1]}, [r0,:16], r3
vst1.16 {d2[1]}, [r0,:16], r3
vst1.16 {d3[1]}, [r0,:16], r3
vst1.16 {d0[2]}, [r0,:16], r3
vst1.16 {d1[2]}, [r0,:16], r3
vst1.16 {d2[2]}, [r0,:16], r3
vst1.16 {d3[2]}, [r0,:16], r3
vst1.16 {d0[3]}, [r0,:16], r3
vst1.16 {d1[3]}, [r0,:16], r3
vst1.16 {d2[3]}, [r0,:16], r3
vst1.16 {d3[3]}, [r0,:16], r3
bx lr
endfunc
function ff_vp8_idct_add_neon, export=1
vld1.16 {q0-q1}, [r1,:128]
movw r3, #20091
movt r3, #35468/2
vdup.32 d4, r3
vmull.s16 q12, d1, d4[0]
vmull.s16 q13, d3, d4[0]
vqdmulh.s16 d20, d1, d4[1]
vqdmulh.s16 d23, d3, d4[1]
vshrn.s32 d21, q12, #16
vshrn.s32 d22, q13, #16
vadd.s16 d21, d21, d1
vadd.s16 d22, d22, d3
vadd.s16 d16, d0, d2
vsub.s16 d17, d0, d2
vadd.s16 d18, d21, d23
vsub.s16 d19, d20, d22
vadd.s16 q0, q8, q9
vsub.s16 q1, q8, q9
vtrn.32 d0, d3
vtrn.32 d1, d2
vtrn.16 d0, d1
vtrn.16 d3, d2
vmov.i16 q15, #0
vmull.s16 q12, d1, d4[0]
vst1.16 {q15}, [r1,:128]!
vmull.s16 q13, d2, d4[0]
vst1.16 {q15}, [r1,:128]
vqdmulh.s16 d21, d1, d4[1]
vqdmulh.s16 d23, d2, d4[1]
vshrn.s32 d20, q12, #16
vshrn.s32 d22, q13, #16
vadd.i16 d20, d20, d1
vadd.i16 d22, d22, d2
vadd.i16 d16, d0, d3
vsub.i16 d17, d0, d3
vadd.i16 d18, d20, d23
vld1.32 {d20[]}, [r0,:32], r2
vsub.i16 d19, d21, d22
vld1.32 {d22[]}, [r0,:32], r2
vadd.s16 q0, q8, q9
vld1.32 {d23[]}, [r0,:32], r2
vsub.s16 q1, q8, q9
vld1.32 {d21[]}, [r0,:32], r2
vrshr.s16 q0, q0, #3
vtrn.32 q10, q11
vrshr.s16 q1, q1, #3
sub r0, r0, r2, lsl #2
vtrn.32 d0, d3
vtrn.32 d1, d2
vtrn.16 d0, d1
vtrn.16 d3, d2
vaddw.u8 q0, q0, d20
vaddw.u8 q1, q1, d21
vqmovun.s16 d0, q0
vqmovun.s16 d1, q1
vst1.32 {d0[0]}, [r0,:32], r2
vst1.32 {d0[1]}, [r0,:32], r2
vst1.32 {d1[1]}, [r0,:32], r2
vst1.32 {d1[0]}, [r0,:32], r2
bx lr
endfunc
function ff_vp8_idct_dc_add_neon, export=1
mov r3, #0
ldrsh r12, [r1]
strh r3, [r1]
vdup.16 q1, r12
vrshr.s16 q1, q1, #3
vld1.32 {d0[]}, [r0,:32], r2
vld1.32 {d1[]}, [r0,:32], r2
vld1.32 {d0[1]}, [r0,:32], r2
vld1.32 {d1[1]}, [r0,:32], r2
vaddw.u8 q2, q1, d0
vaddw.u8 q3, q1, d1
sub r0, r0, r2, lsl #2
vqmovun.s16 d0, q2
vqmovun.s16 d1, q3
vst1.32 {d0[0]}, [r0,:32], r2
vst1.32 {d1[0]}, [r0,:32], r2
vst1.32 {d0[1]}, [r0,:32], r2
vst1.32 {d1[1]}, [r0,:32], r2
bx lr
endfunc
function ff_vp8_idct_dc_add4uv_neon, export=1
vmov.i16 d0, #0
mov r3, #32
vld1.16 {d16[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
vld1.16 {d17[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
vld1.16 {d18[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
vld1.16 {d19[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
mov r3, r0
vrshr.s16 q8, q8, #3 @ dc >>= 3
vld1.8 {d0}, [r0,:64], r2
vrshr.s16 q9, q9, #3
vld1.8 {d1}, [r0,:64], r2
vaddw.u8 q10, q8, d0
vld1.8 {d2}, [r0,:64], r2
vaddw.u8 q0, q8, d1
vld1.8 {d3}, [r0,:64], r2
vaddw.u8 q11, q8, d2
vld1.8 {d4}, [r0,:64], r2
vaddw.u8 q1, q8, d3
vld1.8 {d5}, [r0,:64], r2
vaddw.u8 q12, q9, d4
vld1.8 {d6}, [r0,:64], r2
vaddw.u8 q2, q9, d5
vld1.8 {d7}, [r0,:64], r2
vaddw.u8 q13, q9, d6
vqmovun.s16 d20, q10
vaddw.u8 q3, q9, d7
vqmovun.s16 d21, q0
vqmovun.s16 d22, q11
vst1.8 {d20}, [r3,:64], r2
vqmovun.s16 d23, q1
vst1.8 {d21}, [r3,:64], r2
vqmovun.s16 d24, q12
vst1.8 {d22}, [r3,:64], r2
vqmovun.s16 d25, q2
vst1.8 {d23}, [r3,:64], r2
vqmovun.s16 d26, q13
vst1.8 {d24}, [r3,:64], r2
vqmovun.s16 d27, q3
vst1.8 {d25}, [r3,:64], r2
vst1.8 {d26}, [r3,:64], r2
vst1.8 {d27}, [r3,:64], r2
bx lr
endfunc
function ff_vp8_idct_dc_add4y_neon, export=1
vmov.i16 d0, #0
mov r3, #32
vld1.16 {d16[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
vld1.16 {d17[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
vld1.16 {d18[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
vld1.16 {d19[]}, [r1,:16]
vst1.16 {d0[0]}, [r1,:16], r3
vrshr.s16 q8, q8, #3 @ dc >>= 3
vld1.8 {q0}, [r0,:128], r2
vrshr.s16 q9, q9, #3
vld1.8 {q1}, [r0,:128], r2
vaddw.u8 q10, q8, d0
vld1.8 {q2}, [r0,:128], r2
vaddw.u8 q0, q9, d1
vld1.8 {q3}, [r0,:128], r2
vaddw.u8 q11, q8, d2
vaddw.u8 q1, q9, d3
vaddw.u8 q12, q8, d4
vaddw.u8 q2, q9, d5
vaddw.u8 q13, q8, d6
vaddw.u8 q3, q9, d7
sub r0, r0, r2, lsl #2
vqmovun.s16 d20, q10
vqmovun.s16 d21, q0
vqmovun.s16 d22, q11
vqmovun.s16 d23, q1
vqmovun.s16 d24, q12
vst1.8 {q10}, [r0,:128], r2
vqmovun.s16 d25, q2
vst1.8 {q11}, [r0,:128], r2
vqmovun.s16 d26, q13
vst1.8 {q12}, [r0,:128], r2
vqmovun.s16 d27, q3
vst1.8 {q13}, [r0,:128], r2
bx lr
endfunc
@ Register layout:
@ P3..Q3 -> q0..q7
@ flim_E -> q14
@ flim_I -> q15
@ hev_thresh -> r12
@
.macro vp8_loop_filter, inner=0, simple=0
.if \simple
vabd.u8 q9, q3, q4 @ abs(P0-Q0)
vabd.u8 q15, q2, q5 @ abs(P1-Q1)
vqadd.u8 q9, q9, q9 @ abs(P0-Q0) * 2
vshr.u8 q10, q15, #1 @ abs(P1-Q1) / 2
vqadd.u8 q11, q9, q10 @ (abs(P0-Q0)*2) + (abs(P1-Q1)/2)
vmov.i8 q13, #0x80
vcle.u8 q8, q11, q14 @ (abs(P0-Q0)*2) + (abs(P1-Q1)/2) <= flim
.else
@ calculate hev and normal_limit:
vabd.u8 q12, q2, q3 @ abs(P1-P0)
vabd.u8 q13, q5, q4 @ abs(Q1-Q0)
vabd.u8 q10, q0, q1 @ abs(P3-P2)
vabd.u8 q11, q1, q2 @ abs(P2-P1)
vcle.u8 q8, q12, q15 @ abs(P1-P0) <= flim_I
vcle.u8 q9, q13, q15 @ abs(Q1-Q0) <= flim_I
vcle.u8 q10, q10, q15 @ abs(P3-P2) <= flim_I
vcle.u8 q11, q11, q15 @ abs(P2-P1) <= flim_I
vand q8, q8, q9
vabd.u8 q9, q7, q6 @ abs(Q3-Q2)
vand q8, q8, q11
vabd.u8 q11, q6, q5 @ abs(Q2-Q1)
vand q8, q8, q10
vcle.u8 q10, q9, q15 @ abs(Q3-Q2) <= flim_I
vcle.u8 q11, q11, q15 @ abs(Q2-Q1) <= flim_I
vabd.u8 q9, q3, q4 @ abs(P0-Q0)
vabd.u8 q15, q2, q5 @ abs(P1-Q1)
vand q8, q8, q10
vqadd.u8 q9, q9, q9 @ abs(P0-Q0) * 2
vand q8, q8, q11
vshr.u8 q10, q15, #1 @ abs(P1-Q1) / 2
vdup.8 q15, r12 @ hev_thresh
vqadd.u8 q11, q9, q10 @ (abs(P0-Q0)*2) + (abs(P1-Q1)/2)
vcgt.u8 q12, q12, q15 @ abs(P1-P0) > hev_thresh
vcle.u8 q11, q11, q14 @ (abs(P0-Q0)*2) + (abs(P1-Q1)/2) <= flim_E
vcgt.u8 q14, q13, q15 @ abs(Q1-Q0) > hev_thresh
vand q8, q8, q11
vmov.i8 q13, #0x80
vorr q9, q12, q14
.endif
@ at this point:
@ q8: normal_limit
@ q9: hev
@ convert to signed value:
veor q3, q3, q13 @ PS0 = P0 ^ 0x80
veor q4, q4, q13 @ QS0 = Q0 ^ 0x80
vmov.i16 q12, #3
vsubl.s8 q10, d8, d6 @ QS0 - PS0
vsubl.s8 q11, d9, d7 @ (widened to 16 bits)
veor q2, q2, q13 @ PS1 = P1 ^ 0x80
veor q5, q5, q13 @ QS1 = Q1 ^ 0x80
vmul.i16 q10, q10, q12 @ w = 3 * (QS0 - PS0)
vmul.i16 q11, q11, q12
vqsub.s8 q12, q2, q5 @ clamp(PS1-QS1)
vmov.i8 q14, #4
vmov.i8 q15, #3
.if \inner
vand q12, q12, q9 @ if(hev) w += clamp(PS1-QS1)
.endif
vaddw.s8 q10, q10, d24 @ w += clamp(PS1-QS1)
vaddw.s8 q11, q11, d25
vqmovn.s16 d20, q10 @ narrow result back into q10
vqmovn.s16 d21, q11
.if !\inner && !\simple
veor q1, q1, q13 @ PS2 = P2 ^ 0x80
veor q6, q6, q13 @ QS2 = Q2 ^ 0x80
.endif
vand q10, q10, q8 @ w &= normal_limit
@ registers used at this point..
@ q0 -> P3 (don't corrupt)
@ q1-q6 -> PS2-QS2
@ q7 -> Q3 (don't corrupt)
@ q9 -> hev
@ q10 -> w
@ q13 -> #0x80
@ q14 -> #4
@ q15 -> #3
@ q8, q11, q12 -> unused
@ filter_common: is4tap==1
@ c1 = clamp(w + 4) >> 3;
@ c2 = clamp(w + 3) >> 3;
@ Q0 = s2u(QS0 - c1);
@ P0 = s2u(PS0 + c2);
.if \simple
vqadd.s8 q11, q10, q14 @ c1 = clamp((w&hev)+4)
vqadd.s8 q12, q10, q15 @ c2 = clamp((w&hev)+3)
vshr.s8 q11, q11, #3 @ c1 >>= 3
vshr.s8 q12, q12, #3 @ c2 >>= 3
vqsub.s8 q4, q4, q11 @ QS0 = clamp(QS0-c1)
vqadd.s8 q3, q3, q12 @ PS0 = clamp(PS0+c2)
veor q4, q4, q13 @ Q0 = QS0 ^ 0x80
veor q3, q3, q13 @ P0 = PS0 ^ 0x80
veor q5, q5, q13 @ Q1 = QS1 ^ 0x80
veor q2, q2, q13 @ P1 = PS1 ^ 0x80
.elseif \inner
@ the !is4tap case of filter_common, only used for inner blocks
@ c3 = ((c1&~hev) + 1) >> 1;
@ Q1 = s2u(QS1 - c3);
@ P1 = s2u(PS1 + c3);
vqadd.s8 q11, q10, q14 @ c1 = clamp((w&hev)+4)
vqadd.s8 q12, q10, q15 @ c2 = clamp((w&hev)+3)
vshr.s8 q11, q11, #3 @ c1 >>= 3
vshr.s8 q12, q12, #3 @ c2 >>= 3
vqsub.s8 q4, q4, q11 @ QS0 = clamp(QS0-c1)
vqadd.s8 q3, q3, q12 @ PS0 = clamp(PS0+c2)
vbic q11, q11, q9 @ c1 & ~hev
veor q4, q4, q13 @ Q0 = QS0 ^ 0x80
vrshr.s8 q11, q11, #1 @ c3 >>= 1
veor q3, q3, q13 @ P0 = PS0 ^ 0x80
vqsub.s8 q5, q5, q11 @ QS1 = clamp(QS1-c3)
vqadd.s8 q2, q2, q11 @ PS1 = clamp(PS1+c3)
veor q5, q5, q13 @ Q1 = QS1 ^ 0x80
veor q2, q2, q13 @ P1 = PS1 ^ 0x80
.else
vand q12, q10, q9 @ w & hev
vqadd.s8 q11, q12, q14 @ c1 = clamp((w&hev)+4)
vqadd.s8 q12, q12, q15 @ c2 = clamp((w&hev)+3)
vshr.s8 q11, q11, #3 @ c1 >>= 3
vshr.s8 q12, q12, #3 @ c2 >>= 3
vbic q10, q10, q9 @ w &= ~hev
vqsub.s8 q4, q4, q11 @ QS0 = clamp(QS0-c1)
vqadd.s8 q3, q3, q12 @ PS0 = clamp(PS0+c2)
@ filter_mbedge:
@ a = clamp((27*w + 63) >> 7);
@ Q0 = s2u(QS0 - a);
@ P0 = s2u(PS0 + a);
@ a = clamp((18*w + 63) >> 7);
@ Q1 = s2u(QS1 - a);
@ P1 = s2u(PS1 + a);
@ a = clamp((9*w + 63) >> 7);
@ Q2 = s2u(QS2 - a);
@ P2 = s2u(PS2 + a);
vmov.i16 q9, #63
vshll.s8 q14, d20, #3
vshll.s8 q15, d21, #3
vaddw.s8 q14, q14, d20
vaddw.s8 q15, q15, d21
vadd.s16 q8, q9, q14
vadd.s16 q9, q9, q15 @ 9*w + 63
vadd.s16 q11, q8, q14
vadd.s16 q12, q9, q15 @ 18*w + 63
vadd.s16 q14, q11, q14
vadd.s16 q15, q12, q15 @ 27*w + 63
vqshrn.s16 d16, q8, #7
vqshrn.s16 d17, q9, #7 @ clamp(( 9*w + 63)>>7)
vqshrn.s16 d22, q11, #7
vqshrn.s16 d23, q12, #7 @ clamp((18*w + 63)>>7)
vqshrn.s16 d28, q14, #7
vqshrn.s16 d29, q15, #7 @ clamp((27*w + 63)>>7)
vqadd.s8 q1, q1, q8 @ PS2 = clamp(PS2+a)
vqsub.s8 q6, q6, q8 @ QS2 = clamp(QS2-a)
vqadd.s8 q2, q2, q11 @ PS1 = clamp(PS1+a)
vqsub.s8 q5, q5, q11 @ QS1 = clamp(QS1-a)
vqadd.s8 q3, q3, q14 @ PS0 = clamp(PS0+a)
vqsub.s8 q4, q4, q14 @ QS0 = clamp(QS0-a)
veor q3, q3, q13 @ P0 = PS0 ^ 0x80
veor q4, q4, q13 @ Q0 = QS0 ^ 0x80
veor q2, q2, q13 @ P1 = PS1 ^ 0x80
veor q5, q5, q13 @ Q1 = QS1 ^ 0x80
veor q1, q1, q13 @ P2 = PS2 ^ 0x80
veor q6, q6, q13 @ Q2 = QS2 ^ 0x80
.endif
.endm
.macro vp8_v_loop_filter16 name, inner=0, simple=0
function ff_vp8_v_loop_filter16\name\()_neon, export=1
vpush {q4-q7}
sub r0, r0, r1, lsl #1+!\simple
@ Load pixels:
.if !\simple
ldr r12, [sp, #64] @ hev_thresh
vld1.8 {q0}, [r0,:128], r1 @ P3
vld1.8 {q1}, [r0,:128], r1 @ P2
.endif
vld1.8 {q2}, [r0,:128], r1 @ P1
vld1.8 {q3}, [r0,:128], r1 @ P0
vld1.8 {q4}, [r0,:128], r1 @ Q0
vld1.8 {q5}, [r0,:128], r1 @ Q1
.if !\simple
vld1.8 {q6}, [r0,:128], r1 @ Q2
vld1.8 {q7}, [r0,:128] @ Q3
vdup.8 q15, r3 @ flim_I
.endif
vdup.8 q14, r2 @ flim_E
vp8_loop_filter inner=\inner, simple=\simple
@ back up to P2: dst -= stride * 6
sub r0, r0, r1, lsl #2
.if !\simple
sub r0, r0, r1, lsl #1
@ Store pixels:
vst1.8 {q1}, [r0,:128], r1 @ P2
.endif
vst1.8 {q2}, [r0,:128], r1 @ P1
vst1.8 {q3}, [r0,:128], r1 @ P0
vst1.8 {q4}, [r0,:128], r1 @ Q0
vst1.8 {q5}, [r0,:128], r1 @ Q1
.if !\simple
vst1.8 {q6}, [r0,:128] @ Q2
.endif
vpop {q4-q7}
bx lr
endfunc
.endm
vp8_v_loop_filter16
vp8_v_loop_filter16 _inner, inner=1
vp8_v_loop_filter16 _simple, simple=1
.macro vp8_v_loop_filter8uv name, inner=0
function ff_vp8_v_loop_filter8uv\name\()_neon, export=1
vpush {q4-q7}
sub r0, r0, r2, lsl #2
sub r1, r1, r2, lsl #2
ldr r12, [sp, #64] @ flim_I
@ Load pixels:
vld1.8 {d0}, [r0,:64], r2 @ P3
vld1.8 {d1}, [r1,:64], r2 @ P3
vld1.8 {d2}, [r0,:64], r2 @ P2
vld1.8 {d3}, [r1,:64], r2 @ P2
vld1.8 {d4}, [r0,:64], r2 @ P1
vld1.8 {d5}, [r1,:64], r2 @ P1
vld1.8 {d6}, [r0,:64], r2 @ P0
vld1.8 {d7}, [r1,:64], r2 @ P0
vld1.8 {d8}, [r0,:64], r2 @ Q0
vld1.8 {d9}, [r1,:64], r2 @ Q0
vld1.8 {d10}, [r0,:64], r2 @ Q1
vld1.8 {d11}, [r1,:64], r2 @ Q1
vld1.8 {d12}, [r0,:64], r2 @ Q2
vld1.8 {d13}, [r1,:64], r2 @ Q2
vld1.8 {d14}, [r0,:64] @ Q3
vld1.8 {d15}, [r1,:64] @ Q3
vdup.8 q14, r3 @ flim_E
vdup.8 q15, r12 @ flim_I
ldr r12, [sp, #68] @ hev_thresh
vp8_loop_filter inner=\inner
@ back up to P2: u,v -= stride * 6
sub r0, r0, r2, lsl #2
sub r1, r1, r2, lsl #2
sub r0, r0, r2, lsl #1
sub r1, r1, r2, lsl #1
@ Store pixels:
vst1.8 {d2}, [r0,:64], r2 @ P2
vst1.8 {d3}, [r1,:64], r2 @ P2
vst1.8 {d4}, [r0,:64], r2 @ P1
vst1.8 {d5}, [r1,:64], r2 @ P1
vst1.8 {d6}, [r0,:64], r2 @ P0
vst1.8 {d7}, [r1,:64], r2 @ P0
vst1.8 {d8}, [r0,:64], r2 @ Q0
vst1.8 {d9}, [r1,:64], r2 @ Q0
vst1.8 {d10}, [r0,:64], r2 @ Q1
vst1.8 {d11}, [r1,:64], r2 @ Q1
vst1.8 {d12}, [r0,:64] @ Q2
vst1.8 {d13}, [r1,:64] @ Q2
vpop {q4-q7}
bx lr
endfunc
.endm
vp8_v_loop_filter8uv
vp8_v_loop_filter8uv _inner, inner=1
.macro vp8_h_loop_filter16 name, inner=0, simple=0
function ff_vp8_h_loop_filter16\name\()_neon, export=1
vpush {q4-q7}
sub r0, r0, #4
.if !\simple
ldr r12, [sp, #64] @ hev_thresh
.endif
@ Load pixels:
vld1.8 {d0}, [r0], r1 @ load first 8-line src data
vld1.8 {d2}, [r0], r1
vld1.8 {d4}, [r0], r1
vld1.8 {d6}, [r0], r1
vld1.8 {d8}, [r0], r1
vld1.8 {d10}, [r0], r1
vld1.8 {d12}, [r0], r1
vld1.8 {d14}, [r0], r1
vld1.8 {d1}, [r0], r1 @ load second 8-line src data
vld1.8 {d3}, [r0], r1
vld1.8 {d5}, [r0], r1
vld1.8 {d7}, [r0], r1
vld1.8 {d9}, [r0], r1
vld1.8 {d11}, [r0], r1
vld1.8 {d13}, [r0], r1
vld1.8 {d15}, [r0], r1
transpose_8x8 q0, q1, q2, q3, q4, q5, q6, q7
vdup.8 q14, r2 @ flim_E
.if !\simple
vdup.8 q15, r3 @ flim_I
.endif
vp8_loop_filter inner=\inner, simple=\simple
sub r0, r0, r1, lsl #4 @ backup 16 rows
transpose_8x8 q0, q1, q2, q3, q4, q5, q6, q7
@ Store pixels:
vst1.8 {d0}, [r0], r1
vst1.8 {d2}, [r0], r1
vst1.8 {d4}, [r0], r1
vst1.8 {d6}, [r0], r1
vst1.8 {d8}, [r0], r1
vst1.8 {d10}, [r0], r1
vst1.8 {d12}, [r0], r1
vst1.8 {d14}, [r0], r1
vst1.8 {d1}, [r0], r1
vst1.8 {d3}, [r0], r1
vst1.8 {d5}, [r0], r1
vst1.8 {d7}, [r0], r1
vst1.8 {d9}, [r0], r1
vst1.8 {d11}, [r0], r1
vst1.8 {d13}, [r0], r1
vst1.8 {d15}, [r0]
vpop {q4-q7}
bx lr
endfunc
.endm
vp8_h_loop_filter16
vp8_h_loop_filter16 _inner, inner=1
vp8_h_loop_filter16 _simple, simple=1
.macro vp8_h_loop_filter8uv name, inner=0
function ff_vp8_h_loop_filter8uv\name\()_neon, export=1
vpush {q4-q7}
sub r0, r0, #4
sub r1, r1, #4
ldr r12, [sp, #64] @ flim_I
@ Load pixels:
vld1.8 {d0}, [r0], r2 @ load u
vld1.8 {d1}, [r1], r2 @ load v
vld1.8 {d2}, [r0], r2
vld1.8 {d3}, [r1], r2
vld1.8 {d4}, [r0], r2
vld1.8 {d5}, [r1], r2
vld1.8 {d6}, [r0], r2
vld1.8 {d7}, [r1], r2
vld1.8 {d8}, [r0], r2
vld1.8 {d9}, [r1], r2
vld1.8 {d10}, [r0], r2
vld1.8 {d11}, [r1], r2
vld1.8 {d12}, [r0], r2
vld1.8 {d13}, [r1], r2
vld1.8 {d14}, [r0], r2
vld1.8 {d15}, [r1], r2
transpose_8x8 q0, q1, q2, q3, q4, q5, q6, q7
vdup.8 q14, r3 @ flim_E
vdup.8 q15, r12 @ flim_I
ldr r12, [sp, #68] @ hev_thresh
vp8_loop_filter inner=\inner
sub r0, r0, r2, lsl #3 @ backup u 8 rows
sub r1, r1, r2, lsl #3 @ backup v 8 rows
transpose_8x8 q0, q1, q2, q3, q4, q5, q6, q7
@ Store pixels:
vst1.8 {d0}, [r0], r2
vst1.8 {d1}, [r1], r2
vst1.8 {d2}, [r0], r2
vst1.8 {d3}, [r1], r2
vst1.8 {d4}, [r0], r2
vst1.8 {d5}, [r1], r2
vst1.8 {d6}, [r0], r2
vst1.8 {d7}, [r1], r2
vst1.8 {d8}, [r0], r2
vst1.8 {d9}, [r1], r2
vst1.8 {d10}, [r0], r2
vst1.8 {d11}, [r1], r2
vst1.8 {d12}, [r0], r2
vst1.8 {d13}, [r1], r2
vst1.8 {d14}, [r0]
vst1.8 {d15}, [r1]
vpop {q4-q7}
bx lr
endfunc
.endm
vp8_h_loop_filter8uv
vp8_h_loop_filter8uv _inner, inner=1
function ff_put_vp8_pixels16_neon, export=1
ldr r12, [sp, #0] @ h
1:
subs r12, r12, #4
vld1.8 {q0}, [r2], r3
vld1.8 {q1}, [r2], r3
vld1.8 {q2}, [r2], r3
vld1.8 {q3}, [r2], r3
vst1.8 {q0}, [r0,:128], r1
vst1.8 {q1}, [r0,:128], r1
vst1.8 {q2}, [r0,:128], r1
vst1.8 {q3}, [r0,:128], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_pixels8_neon, export=1
ldr r12, [sp, #0] @ h
1:
subs r12, r12, #4
vld1.8 {d0}, [r2], r3
vld1.8 {d1}, [r2], r3
vld1.8 {d2}, [r2], r3
vld1.8 {d3}, [r2], r3
vst1.8 {d0}, [r0,:64], r1
vst1.8 {d1}, [r0,:64], r1
vst1.8 {d2}, [r0,:64], r1
vst1.8 {d3}, [r0,:64], r1
bgt 1b
bx lr
endfunc
.macro vp8_epel8_h6 d, a, b
vext.8 d27, \a, \b, #1
vmovl.u8 q8, \a
vext.8 d28, \a, \b, #2
vmovl.u8 q9, d27
vext.8 d29, \a, \b, #3
vmovl.u8 q10, d28
vext.8 d30, \a, \b, #4
vmovl.u8 q11, d29
vext.8 d31, \a, \b, #5
vmovl.u8 q12, d30
vmul.u16 q10, q10, d0[2]
vmovl.u8 q13, d31
vmul.u16 q11, q11, d0[3]
vmls.u16 q10, q9, d0[1]
vmls.u16 q11, q12, d1[0]
vmla.u16 q10, q8, d0[0]
vmla.u16 q11, q13, d1[1]
vqadd.s16 q11, q10, q11
vqrshrun.s16 \d, q11, #7
.endm
.macro vp8_epel16_h6 d0, d1, s0, s1, s2, q0, q1
vext.8 q14, \q0, \q1, #3
vext.8 q15, \q0, \q1, #4
vmovl.u8 q11, d28
vmovl.u8 q14, d29
vext.8 q3, \q0, \q1, #2
vmovl.u8 q12, d30
vmovl.u8 q15, d31
vext.8 q8, \q0, \q1, #1
vmovl.u8 q10, d6
vmovl.u8 q3, d7
vext.8 q2, \q0, \q1, #5
vmovl.u8 q13, d4
vmovl.u8 q2, d5
vmovl.u8 q9, d16
vmovl.u8 q8, d17
vmul.u16 q11, q11, d0[3]
vmul.u16 q10, q10, d0[2]
vmul.u16 q3, q3, d0[2]
vmul.u16 q14, q14, d0[3]
vmls.u16 q11, q12, d1[0]
vmovl.u8 q12, \s0
vmovl.u8 q1, \s1
vmls.u16 q10, q9, d0[1]
vmls.u16 q3, q8, d0[1]
vmls.u16 q14, q15, d1[0]
vmla.u16 q10, q12, d0[0]
vmla.u16 q11, q13, d1[1]
vmla.u16 q3, q1, d0[0]
vmla.u16 q14, q2, d1[1]
vqadd.s16 q11, q10, q11
vqadd.s16 q14, q3, q14
vqrshrun.s16 \d0, q11, #7
vqrshrun.s16 \d1, q14, #7
.endm
.macro vp8_epel8_v6_y2 d0, d1, s0, s1, s2, s3, s4, s5, s6
vmovl.u8 q10, \s0
vmovl.u8 q11, \s3
vmovl.u8 q14, \s6
vmovl.u8 q9, \s1
vmovl.u8 q12, \s4
vmovl.u8 q8, \s2
vmovl.u8 q13, \s5
vmul.u16 q10, q10, d0[0]
vmul.u16 q15, q11, d0[3]
vmul.u16 q11, q11, d0[2]
vmul.u16 q14, q14, d1[1]
vmls.u16 q10, q9, d0[1]
vmls.u16 q15, q12, d1[0]
vmls.u16 q11, q8, d0[1]
vmls.u16 q14, q13, d1[0]
vmla.u16 q10, q8, d0[2]
vmla.u16 q15, q13, d1[1]
vmla.u16 q11, q9, d0[0]
vmla.u16 q14, q12, d0[3]
vqadd.s16 q15, q10, q15
vqadd.s16 q14, q11, q14
vqrshrun.s16 \d0, q15, #7
vqrshrun.s16 \d1, q14, #7
.endm
.macro vp8_epel8_h4 d, a, b
vext.8 d28, \a, \b, #1
vmovl.u8 q9, \a
vext.8 d29, \a, \b, #2
vmovl.u8 q10, d28
vext.8 d30, \a, \b, #3
vmovl.u8 q11, d29
vmovl.u8 q12, d30
vmul.u16 q10, q10, d0[2]
vmul.u16 q11, q11, d0[3]
vmls.u16 q10, q9, d0[1]
vmls.u16 q11, q12, d1[0]
vqadd.s16 q11, q10, q11
vqrshrun.s16 \d, q11, #7
.endm
.macro vp8_epel8_v4_y2 d0, d1, s0, s1, s2, s3, s4
vmovl.u8 q9, \s0
vmovl.u8 q10, \s1
vmovl.u8 q11, \s2
vmovl.u8 q12, \s3
vmovl.u8 q13, \s4
vmul.u16 q8, q10, d0[2]
vmul.u16 q14, q11, d0[3]
vmul.u16 q11, q11, d0[2]
vmul.u16 q15, q12, d0[3]
vmls.u16 q8, q9, d0[1]
vmls.u16 q14, q12, d1[0]
vmls.u16 q11, q10, d0[1]
vmls.u16 q15, q13, d1[0]
vqadd.s16 q8, q8, q14
vqadd.s16 q11, q11, q15
vqrshrun.s16 \d0, q8, #7
vqrshrun.s16 \d1, q11, #7
.endm
function ff_put_vp8_epel16_v6_neon, export=1
sub r2, r2, r3, lsl #1
push {r4,lr}
vpush {d8-d15}
ldr r4, [sp, #80] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #72] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {d2-d3}, [r2], r3
vld1.8 {d4-d5}, [r2], r3
vld1.8 {d6-d7}, [r2], r3
vld1.8 {d8-d9}, [r2], r3
vld1.8 {d10-d11},[r2], r3
vld1.8 {d12-d13},[r2], r3
vld1.8 {d14-d15},[r2]
sub r2, r2, r3, lsl #2
vp8_epel8_v6_y2 d2, d4, d2, d4, d6, d8, d10, d12, d14
vp8_epel8_v6_y2 d3, d5, d3, d5, d7, d9, d11, d13, d15
vst1.8 {d2-d3}, [r0,:128], r1
vst1.8 {d4-d5}, [r0,:128], r1
subs r12, r12, #2
bne 1b
vpop {d8-d15}
pop {r4,pc}
endfunc
function ff_put_vp8_epel16_h6_neon, export=1
sub r2, r2, #2
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {d2-d4}, [r2], r3
vp8_epel16_h6 d2, d3, d2, d3, d4, q1, q2
vst1.8 {d2-d3}, [r0,:128], r1
subs r12, r12, #1
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel16_h6v6_neon, export=1
sub r2, r2, r3, lsl #1
sub r2, r2, #2
push {r4,lr}
vpush {d8-d15}
@ first pass (horizontal):
ldr r4, [sp, #64+8+4] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #64+8+0] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #336+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #5
bic lr, lr, #15
1:
vld1.8 {d2,d3,d4}, [r2], r3
vp8_epel16_h6 d2, d3, d2, d3, d4, q1, q2
vst1.8 {d2-d3}, [lr,:128]!
subs r12, r12, #1
bne 1b
@ second pass (vertical):
ldr r4, [sp, #336+16+64+8+8] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #336+16+64+8+0] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d5}, [lr,:128]!
vld1.8 {d6-d9}, [lr,:128]!
vld1.8 {d10-d13},[lr,:128]!
vld1.8 {d14-d15},[lr,:128]
sub lr, lr, #64
vp8_epel8_v6_y2 d2, d4, d2, d4, d6, d8, d10, d12, d14
vp8_epel8_v6_y2 d3, d5, d3, d5, d7, d9, d11, d13, d15
vst1.8 {d2-d3}, [r0,:128], r1
vst1.8 {d4-d5}, [r0,:128], r1
subs r12, r12, #2
bne 2b
add sp, sp, #336+16
vpop {d8-d15}
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_v6_neon, export=1
sub r2, r2, r3, lsl #1
push {r4,lr}
ldr r4, [sp, #16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {d2}, [r2], r3
vld1.8 {d3}, [r2], r3
vld1.8 {d4}, [r2], r3
vld1.8 {d5}, [r2], r3
vld1.8 {d6}, [r2], r3
vld1.8 {d7}, [r2], r3
vld1.8 {d28}, [r2]
sub r2, r2, r3, lsl #2
vp8_epel8_v6_y2 d2, d3, d2, d3, d4, d5, d6, d7, d28
vst1.8 {d2}, [r0,:64], r1
vst1.8 {d3}, [r0,:64], r1
subs r12, r12, #2
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_h6_neon, export=1
sub r2, r2, #2
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {d2,d3}, [r2], r3
vp8_epel8_h6 d2, d2, d3
vst1.8 {d2}, [r0,:64], r1
subs r12, r12, #1
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_h6v6_neon, export=1
sub r2, r2, r3, lsl #1
sub r2, r2, #2
push {r4,lr}
@ first pass (horizontal):
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #168+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #5
bic lr, lr, #15
1:
vld1.8 {d2,d3}, [r2], r3
vp8_epel8_h6 d2, d2, d3
vst1.8 {d2}, [lr,:64]!
subs r12, r12, #1
bne 1b
@ second pass (vertical):
ldr r4, [sp, #168+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #168+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d5}, [lr,:128]!
vld1.8 {d6-d7}, [lr,:128]!
vld1.8 {d30}, [lr,:64]
sub lr, lr, #32
vp8_epel8_v6_y2 d2, d3, d2, d3, d4, d5, d6, d7, d30
vst1.8 {d2}, [r0,:64], r1
vst1.8 {d3}, [r0,:64], r1
subs r12, r12, #2
bne 2b
add sp, sp, #168+16
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_v4_neon, export=1
sub r2, r2, r3
push {r4,lr}
ldr r4, [sp, #16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {d2}, [r2], r3
vld1.8 {d3}, [r2], r3
vld1.8 {d4}, [r2], r3
vld1.8 {d5}, [r2], r3
vld1.8 {d6}, [r2]
sub r2, r2, r3, lsl #1
vp8_epel8_v4_y2 d2, d3, d2, d3, d4, d5, d6
vst1.8 {d2}, [r0,:64], r1
vst1.8 {d3}, [r0,:64], r1
subs r12, r12, #2
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_h4_neon, export=1
sub r2, r2, #1
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {d2,d3}, [r2], r3
vp8_epel8_h4 d2, d2, d3
vst1.8 {d2}, [r0,:64], r1
subs r12, r12, #1
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_h4v4_neon, export=1
sub r2, r2, r3
sub r2, r2, #1
push {r4,lr}
@ first pass (horizontal):
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #168+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #3
bic lr, lr, #15
1:
vld1.8 {d2,d3}, [r2], r3
vp8_epel8_h4 d2, d2, d3
vst1.8 {d2}, [lr,:64]!
subs r12, r12, #1
bne 1b
@ second pass (vertical):
ldr r4, [sp, #168+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #168+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d5}, [lr,:128]!
vld1.8 {d6}, [lr,:64]
sub lr, lr, #16
vp8_epel8_v4_y2 d2, d3, d2, d3, d4, d5, d6
vst1.8 {d2}, [r0,:64], r1
vst1.8 {d3}, [r0,:64], r1
subs r12, r12, #2
bne 2b
add sp, sp, #168+16
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_h6v4_neon, export=1
sub r2, r2, r3
sub r2, r2, #2
push {r4,lr}
@ first pass (horizontal):
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #168+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #3
bic lr, lr, #15
1:
vld1.8 {d2,d3}, [r2], r3
vp8_epel8_h6 d2, d2, d3
vst1.8 {d2}, [lr,:64]!
subs r12, r12, #1
bne 1b
@ second pass (vertical):
ldr r4, [sp, #168+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #168+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d5}, [lr,:128]!
vld1.8 {d6}, [lr,:64]
sub lr, lr, #16
vp8_epel8_v4_y2 d2, d3, d2, d3, d4, d5, d6
vst1.8 {d2}, [r0,:64], r1
vst1.8 {d3}, [r0,:64], r1
subs r12, r12, #2
bne 2b
add sp, sp, #168+16
pop {r4,pc}
endfunc
function ff_put_vp8_epel8_h4v6_neon, export=1
sub r2, r2, r3, lsl #1
sub r2, r2, #1
push {r4,lr}
@ first pass (horizontal):
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #168+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #5
bic lr, lr, #15
1:
vld1.8 {d2,d3}, [r2], r3
vp8_epel8_h4 d2, d2, d3
vst1.8 {d2}, [lr,:64]!
subs r12, r12, #1
bne 1b
@ second pass (vertical):
ldr r4, [sp, #168+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #168+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d5}, [lr,:128]!
vld1.8 {d6-d7}, [lr,:128]!
vld1.8 {d30}, [lr,:64]
sub lr, lr, #32
vp8_epel8_v6_y2 d2, d3, d2, d3, d4, d5, d6, d7, d30
vst1.8 {d2}, [r0,:64], r1
vst1.8 {d3}, [r0,:64], r1
subs r12, r12, #2
bne 2b
add sp, sp, #168+16
pop {r4,pc}
endfunc
.ltorg
function ff_put_vp8_epel4_v6_neon, export=1
sub r2, r2, r3, lsl #1
push {r4,lr}
ldr r4, [sp, #16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.32 {d2[]}, [r2], r3
vld1.32 {d3[]}, [r2], r3
vld1.32 {d4[]}, [r2], r3
vld1.32 {d5[]}, [r2], r3
vld1.32 {d6[]}, [r2], r3
vld1.32 {d7[]}, [r2], r3
vld1.32 {d28[]}, [r2]
sub r2, r2, r3, lsl #2
vld1.32 {d2[1]}, [r2], r3
vld1.32 {d3[1]}, [r2], r3
vld1.32 {d4[1]}, [r2], r3
vld1.32 {d5[1]}, [r2], r3
vld1.32 {d6[1]}, [r2], r3
vld1.32 {d7[1]}, [r2], r3
vld1.32 {d28[1]}, [r2]
sub r2, r2, r3, lsl #2
vp8_epel8_v6_y2 d2, d3, d2, d3, d4, d5, d6, d7, d28
vst1.32 {d2[0]}, [r0,:32], r1
vst1.32 {d3[0]}, [r0,:32], r1
vst1.32 {d2[1]}, [r0,:32], r1
vst1.32 {d3[1]}, [r0,:32], r1
subs r12, r12, #4
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel4_h6_neon, export=1
sub r2, r2, #2
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {q1}, [r2], r3
vp8_epel8_h6 d2, d2, d3
vst1.32 {d2[0]}, [r0,:32], r1
subs r12, r12, #1
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel4_h6v6_neon, export=1
sub r2, r2, r3, lsl #1
sub r2, r2, #2
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #52+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #5
bic lr, lr, #15
1:
vld1.8 {q1}, [r2], r3
vp8_epel8_h6 d2, d2, d3
vst1.32 {d2[0]}, [lr,:32]!
subs r12, r12, #1
bne 1b
ldr r4, [sp, #52+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #52+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d3}, [lr,:128]!
vld1.8 {d6}, [lr,:64]!
vld1.32 {d28[]}, [lr,:32]
sub lr, lr, #16
vld1.8 {d4-d5}, [lr]!
vld1.8 {d7}, [lr,:64]!
vld1.32 {d28[1]}, [lr,:32]
sub lr, lr, #16
vtrn.32 q1, q2
vtrn.32 d6, d7
vp8_epel8_v6_y2 d2, d3, d2, d4, d3, d5, d6, d7, d28
vst1.32 {d2[0]}, [r0,:32], r1
vst1.32 {d3[0]}, [r0,:32], r1
vst1.32 {d2[1]}, [r0,:32], r1
vst1.32 {d3[1]}, [r0,:32], r1
subs r12, r12, #4
bne 2b
add sp, sp, #52+16
pop {r4,pc}
endfunc
function ff_put_vp8_epel4_h4v6_neon, export=1
sub r2, r2, r3, lsl #1
sub r2, r2, #1
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #52+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #5
bic lr, lr, #15
1:
vld1.8 {d2}, [r2], r3
vp8_epel8_h4 d2, d2, d2
vst1.32 {d2[0]}, [lr,:32]!
subs r12, r12, #1
bne 1b
ldr r4, [sp, #52+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #52+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d3}, [lr,:128]!
vld1.8 {d6}, [lr,:64]!
vld1.32 {d28[]}, [lr,:32]
sub lr, lr, #16
vld1.8 {d4-d5}, [lr]!
vld1.8 {d7}, [lr,:64]!
vld1.32 {d28[1]}, [lr,:32]
sub lr, lr, #16
vtrn.32 q1, q2
vtrn.32 d6, d7
vp8_epel8_v6_y2 d2, d3, d2, d4, d3, d5, d6, d7, d28
vst1.32 {d2[0]}, [r0,:32], r1
vst1.32 {d3[0]}, [r0,:32], r1
vst1.32 {d2[1]}, [r0,:32], r1
vst1.32 {d3[1]}, [r0,:32], r1
subs r12, r12, #4
bne 2b
add sp, sp, #52+16
pop {r4,pc}
endfunc
function ff_put_vp8_epel4_h6v4_neon, export=1
sub r2, r2, r3
sub r2, r2, #2
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #44+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #3
bic lr, lr, #15
1:
vld1.8 {q1}, [r2], r3
vp8_epel8_h6 d2, d2, d3
vst1.32 {d2[0]}, [lr,:32]!
subs r12, r12, #1
bne 1b
ldr r4, [sp, #44+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #44+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d3}, [lr,:128]!
vld1.32 {d6[]}, [lr,:32]
sub lr, lr, #8
vld1.8 {d4-d5}, [lr]!
vld1.32 {d6[1]}, [lr,:32]
sub lr, lr, #8
vtrn.32 q1, q2
vp8_epel8_v4_y2 d2, d3, d2, d4, d3, d5, d6
vst1.32 {d2[0]}, [r0,:32], r1
vst1.32 {d3[0]}, [r0,:32], r1
vst1.32 {d2[1]}, [r0,:32], r1
vst1.32 {d3[1]}, [r0,:32], r1
subs r12, r12, #4
bne 2b
add sp, sp, #44+16
pop {r4,pc}
endfunc
function ff_put_vp8_epel4_h4_neon, export=1
sub r2, r2, #1
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.8 {d2}, [r2], r3
vp8_epel8_h4 d2, d2, d2
vst1.32 {d2[0]}, [r0,:32], r1
subs r12, r12, #1
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel4_v4_neon, export=1
sub r2, r2, r3
push {r4,lr}
ldr r4, [sp, #16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
vld1.16 {q0}, [r4,:128]
1:
vld1.32 {d2[]}, [r2], r3
vld1.32 {d3[]}, [r2], r3
vld1.32 {d4[]}, [r2], r3
vld1.32 {d5[]}, [r2], r3
vld1.32 {d6[]}, [r2]
sub r2, r2, r3, lsl #1
vld1.32 {d2[1]}, [r2], r3
vld1.32 {d3[1]}, [r2], r3
vld1.32 {d4[1]}, [r2], r3
vld1.32 {d5[1]}, [r2], r3
vld1.32 {d6[1]}, [r2]
sub r2, r2, r3, lsl #1
vp8_epel8_v4_y2 d2, d3, d2, d3, d4, d5, d6
vst1.32 {d2[0]}, [r0,:32], r1
vst1.32 {d3[0]}, [r0,:32], r1
vst1.32 {d2[1]}, [r0,:32], r1
vst1.32 {d3[1]}, [r0,:32], r1
subs r12, r12, #4
bne 1b
pop {r4,pc}
endfunc
function ff_put_vp8_epel4_h4v4_neon, export=1
sub r2, r2, r3
sub r2, r2, #1
push {r4,lr}
ldr r4, [sp, #12] @ mx
movrel lr, subpel_filters-16
ldr r12, [sp, #8] @ h
add r4, lr, r4, lsl #4
sub sp, sp, #44+16
vld1.16 {q0}, [r4,:128]
add lr, sp, #15
add r12, r12, #3
bic lr, lr, #15
1:
vld1.8 {d2}, [r2], r3
vp8_epel8_h4 d2, d2, d3
vst1.32 {d2[0]}, [lr,:32]!
subs r12, r12, #1
bne 1b
ldr r4, [sp, #44+16+16] @ my
movrel lr, subpel_filters-16
ldr r12, [sp, #44+16+8] @ h
add r4, lr, r4, lsl #4
add lr, sp, #15
vld1.16 {q0}, [r4,:128]
bic lr, lr, #15
2:
vld1.8 {d2-d3}, [lr,:128]!
vld1.32 {d6[]}, [lr,:32]
sub lr, lr, #8
vld1.8 {d4-d5}, [lr]!
vld1.32 {d6[1]}, [lr,:32]
sub lr, lr, #8
vtrn.32 q1, q2
vp8_epel8_v4_y2 d2, d3, d2, d4, d3, d5, d6
vst1.32 {d2[0]}, [r0,:32], r1
vst1.32 {d3[0]}, [r0,:32], r1
vst1.32 {d2[1]}, [r0,:32], r1
vst1.32 {d3[1]}, [r0,:32], r1
subs r12, r12, #4
bne 2b
add sp, sp, #44+16
pop {r4,pc}
endfunc
@ note: worst case sum of all 6-tap filter values * 255 is 0x7f80 so 16 bit
@ arithmetic can be used to apply filters
const subpel_filters, align=4
.short 0, 6, 123, 12, 1, 0, 0, 0
.short 2, 11, 108, 36, 8, 1, 0, 0
.short 0, 9, 93, 50, 6, 0, 0, 0
.short 3, 16, 77, 77, 16, 3, 0, 0
.short 0, 6, 50, 93, 9, 0, 0, 0
.short 1, 8, 36, 108, 11, 2, 0, 0
.short 0, 1, 12, 123, 6, 0, 0, 0
endconst
function ff_put_vp8_bilin16_h_neon, export=1
ldr r12, [sp, #4] @ mx
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp] @ h
1:
subs r12, r12, #2
vld1.8 {d2-d4}, [r2], r3
vext.8 q2, q1, q2, #1
vmull.u8 q8, d2, d1
vmlal.u8 q8, d4, d0
vld1.8 {d18-d20},[r2], r3
vmull.u8 q3, d3, d1
vmlal.u8 q3, d5, d0
vext.8 q10, q9, q10, #1
vmull.u8 q11, d18, d1
vmlal.u8 q11, d20, d0
vmull.u8 q12, d19, d1
vmlal.u8 q12, d21, d0
vrshrn.u16 d4, q8, #3
vrshrn.u16 d5, q3, #3
vrshrn.u16 d6, q11, #3
vrshrn.u16 d7, q12, #3
vst1.8 {q2}, [r0,:128], r1
vst1.8 {q3}, [r0,:128], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin16_v_neon, export=1
ldr r12, [sp, #8] @ my
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp] @ h
vld1.8 {q1}, [r2], r3
1:
subs r12, r12, #2
vld1.8 {q2}, [r2], r3
vmull.u8 q3, d2, d1
vmlal.u8 q3, d4, d0
vmull.u8 q8, d3, d1
vmlal.u8 q8, d5, d0
vld1.8 {q1}, [r2], r3
vmull.u8 q9, d4, d1
vmlal.u8 q9, d2, d0
vmull.u8 q10, d5, d1
vmlal.u8 q10, d3, d0
vrshrn.u16 d4, q3, #3
vrshrn.u16 d5, q8, #3
vrshrn.u16 d6, q9, #3
vrshrn.u16 d7, q10, #3
vst1.8 {q2}, [r0,:128], r1
vst1.8 {q3}, [r0,:128], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin16_hv_neon, export=1
ldr r12, [sp, #4] @ mx
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp, #8] @ my
vdup.8 d2, r12
rsb r12, r12, #8
vdup.8 d3, r12
ldr r12, [sp] @ h
vld1.8 {d4-d6}, [r2], r3
vext.8 q3, q2, q3, #1
vmull.u8 q8, d4, d1
vmlal.u8 q8, d6, d0
vmull.u8 q9, d5, d1
vmlal.u8 q9, d7, d0
vrshrn.u16 d4, q8, #3
vrshrn.u16 d5, q9, #3
1:
subs r12, r12, #2
vld1.8 {d18-d20},[r2], r3
vext.8 q10, q9, q10, #1
vmull.u8 q11, d18, d1
vmlal.u8 q11, d20, d0
vld1.8 {d26-d28},[r2], r3
vmull.u8 q12, d19, d1
vmlal.u8 q12, d21, d0
vext.8 q14, q13, q14, #1
vmull.u8 q8, d26, d1
vmlal.u8 q8, d28, d0
vmull.u8 q9, d27, d1
vmlal.u8 q9, d29, d0
vrshrn.u16 d6, q11, #3
vrshrn.u16 d7, q12, #3
vmull.u8 q12, d4, d3
vmlal.u8 q12, d6, d2
vmull.u8 q15, d5, d3
vmlal.u8 q15, d7, d2
vrshrn.u16 d4, q8, #3
vrshrn.u16 d5, q9, #3
vmull.u8 q10, d6, d3
vmlal.u8 q10, d4, d2
vmull.u8 q11, d7, d3
vmlal.u8 q11, d5, d2
vrshrn.u16 d24, q12, #3
vrshrn.u16 d25, q15, #3
vst1.8 {q12}, [r0,:128], r1
vrshrn.u16 d20, q10, #3
vrshrn.u16 d21, q11, #3
vst1.8 {q10}, [r0,:128], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin8_h_neon, export=1
ldr r12, [sp, #4] @ mx
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp] @ h
1:
subs r12, r12, #2
vld1.8 {q1}, [r2], r3
vext.8 d3, d2, d3, #1
vmull.u8 q2, d2, d1
vmlal.u8 q2, d3, d0
vld1.8 {q3}, [r2], r3
vext.8 d7, d6, d7, #1
vmull.u8 q8, d6, d1
vmlal.u8 q8, d7, d0
vrshrn.u16 d4, q2, #3
vrshrn.u16 d16, q8, #3
vst1.8 {d4}, [r0,:64], r1
vst1.8 {d16}, [r0,:64], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin8_v_neon, export=1
ldr r12, [sp, #8] @ my
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp] @ h
vld1.8 {d2}, [r2], r3
1:
subs r12, r12, #2
vld1.8 {d3}, [r2], r3
vmull.u8 q2, d2, d1
vmlal.u8 q2, d3, d0
vld1.8 {d2}, [r2], r3
vmull.u8 q3, d3, d1
vmlal.u8 q3, d2, d0
vrshrn.u16 d4, q2, #3
vrshrn.u16 d6, q3, #3
vst1.8 {d4}, [r0,:64], r1
vst1.8 {d6}, [r0,:64], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin8_hv_neon, export=1
ldr r12, [sp, #4] @ mx
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp, #8] @ my
vdup.8 d2, r12
rsb r12, r12, #8
vdup.8 d3, r12
ldr r12, [sp] @ h
vld1.8 {q2}, [r2], r3
vext.8 d5, d4, d5, #1
vmull.u8 q9, d4, d1
vmlal.u8 q9, d5, d0
vrshrn.u16 d22, q9, #3
1:
subs r12, r12, #2
vld1.8 {q3}, [r2], r3
vext.8 d7, d6, d7, #1
vmull.u8 q8, d6, d1
vmlal.u8 q8, d7, d0
vld1.8 {q2}, [r2], r3
vext.8 d5, d4, d5, #1
vmull.u8 q9, d4, d1
vmlal.u8 q9, d5, d0
vrshrn.u16 d16, q8, #3
vmull.u8 q10, d22, d3
vmlal.u8 q10, d16, d2
vrshrn.u16 d22, q9, #3
vmull.u8 q12, d16, d3
vmlal.u8 q12, d22, d2
vrshrn.u16 d20, q10, #3
vst1.8 {d20}, [r0,:64], r1
vrshrn.u16 d23, q12, #3
vst1.8 {d23}, [r0,:64], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin4_h_neon, export=1
ldr r12, [sp, #4] @ mx
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp] @ h
1:
subs r12, r12, #2
vld1.8 {d2}, [r2], r3
vext.8 d3, d2, d3, #1
vld1.8 {d6}, [r2], r3
vext.8 d7, d6, d7, #1
vtrn.32 q1, q3
vmull.u8 q2, d2, d1
vmlal.u8 q2, d3, d0
vrshrn.u16 d4, q2, #3
vst1.32 {d4[0]}, [r0,:32], r1
vst1.32 {d4[1]}, [r0,:32], r1
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin4_v_neon, export=1
ldr r12, [sp, #8] @ my
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp] @ h
vld1.32 {d2[]}, [r2], r3
1:
vld1.32 {d3[]}, [r2]
vld1.32 {d2[1]}, [r2], r3
vld1.32 {d3[1]}, [r2], r3
vmull.u8 q2, d2, d1
vmlal.u8 q2, d3, d0
vtrn.32 d3, d2
vrshrn.u16 d4, q2, #3
vst1.32 {d4[0]}, [r0,:32], r1
vst1.32 {d4[1]}, [r0,:32], r1
subs r12, r12, #2
bgt 1b
bx lr
endfunc
function ff_put_vp8_bilin4_hv_neon, export=1
ldr r12, [sp, #4] @ mx
vdup.8 d0, r12
rsb r12, r12, #8
vdup.8 d1, r12
ldr r12, [sp, #8] @ my
vdup.8 d2, r12
rsb r12, r12, #8
vdup.8 d3, r12
ldr r12, [sp] @ h
vld1.8 {d4}, [r2], r3
vext.8 d5, d4, d4, #1
vmull.u8 q9, d4, d1
vmlal.u8 q9, d5, d0
vrshrn.u16 d22, q9, #3
1:
subs r12, r12, #2
vld1.8 {d6}, [r2], r3
vext.8 d7, d6, d6, #1
vld1.8 {d4}, [r2], r3
vext.8 d5, d4, d4, #1
vtrn.32 q3, q2
vmull.u8 q8, d6, d1
vmlal.u8 q8, d7, d0
vrshrn.u16 d16, q8, #3
vmull.u8 q10, d16, d2
vtrn.32 d22, d16
vmlal.u8 q10, d22, d3
vrev64.32 d22, d16
vrshrn.u16 d20, q10, #3
vst1.32 {d20[0]}, [r0,:32], r1
vst1.32 {d20[1]}, [r0,:32], r1
bgt 1b
bx lr
endfunc