Open Quuxplusone opened 5 years ago
Quuxplusone
commented
5 years ago | Bugzilla Link | PR40490 |
| Status | NEW |
| Importance | P enhancement |
| Reported by | Devin Hussey (husseydevin@gmail.com) |
| Reported on | 2019-01-27 23:37:31 -0800 |
| Last modified on | 2019-01-28 06:38:46 -0800 |
| Version | trunk |
| Hardware | All All |
| CC | hfinkel@anl.gov, llvm-bugs@lists.llvm.org |
| Fixed by commit(s) | |
| Attachments | |
| Blocks | |
| Blocked by | |
| See also |
Quuxplusone
commented
5 years ago Gah, stupid enter button kept submitting my changes before I wanted them to.
LLVM simply cannot vectorize an unrolled loop.
void add_unroll_while_reverse_ptradd(unsigned *__restrict__ p1, const unsigned
*__restrict__ p2)
{
const unsigned *const end = p1 + 1024;
while (p1 != end)
{
*p1++ += *p2++;
*p1++ += *p2++;
*p1++ += *p2++;
*p1++ += *p2++;
}
}
Expected code output (ARM NEON, affects other platforms) would be something
like this:
add_unroll_while_reverse_ptradd:
mov r2, #0
.LBB0_1
add r3, r1, r2
vld1.32 {d16, d17}, [r3]
add r3, r0, r2
add r2, r2, #16
vld1.32 {d18, d19}, [r3]
cmp r2, #4096
vadd.i32 q8, q9, q8
vst1.32 {d16, d17}, [r3]
bne .LBB0_1
bx lr
Actual output:
add_unroll_while_reverse_ptradd:
push {r4, r5, r6, r7, r8, r9, r11, lr}
add r2, r1, #4096
cmp r2, r0
addhi r2, r0, #4096
cmphi r2, r1
bhi .LBB4_3
mov lr, #0
.LBB4_2:
add r2, r1, lr
add r12, r0, lr
add lr, lr, #64
mov r3, r2
mov r4, r12
vld1.32 {d16, d17}, [r3]!
cmp lr, #4096
vld1.32 {d18, d19}, [r4]!
vadd.i32 q8, q9, q8
vld1.32 {d20, d21}, [r3]
add r3, r2, #48
add r2, r2, #32
vld1.32 {d22, d23}, [r3]
add r3, r12, #48
vld1.32 {d26, d27}, [r2]
add r2, r12, #32
vld1.32 {d28, d29}, [r4]
vadd.i32 q9, q14, q10
vld1.32 {d20, d21}, [r2]
vadd.i32 q10, q10, q13
vld1.32 {d24, d25}, [r3]
vorr q13, q8, q8
vadd.i32 q11, q12, q11
vorr q12, q9, q9
vorr q14, q10, q10
vorr q15, q11, q11
vtrn.32 q13, q12
vtrn.32 q14, q15
vorr q14, q10, q10
vext.32 q0, q12, q8, #2
vzip.32 q14, q11
vzip.32 q8, q9
vext.32 q1, q10, q15, #2
vext.32 q8, q10, q11, #2
vext.32 q11, q15, q0, #2
vext.32 q12, q1, q12, #2
vext.32 q3, q11, q11, #2
vext.32 q10, q10, q14, #2
vext.32 q8, q8, q9, #2
vext.32 q1, q12, q12, #2
vext.32 q9, q10, q13, #2
vext.32 q2, q8, q8, #2
vext.32 q0, q9, q9, #2
vst4.32 {d0, d2, d4, d6}, [r12]!
vst4.32 {d1, d3, d5, d7}, [r12]
bne .LBB4_2
b .LBB4_5
.LBB4_3:
mov r2, #0
.LBB4_4:
mov r3, r0
mov r4, r1
ldr r12, [r3, r2, lsl #2]!
ldr lr, [r4, r2, lsl #2]!
add r2, r2, #4
ldmib r3, {r8, r9}
cmp r2, #1024
add r12, r12, lr
ldr r7, [r3, #12]
ldmib r4, {r5, r6}
ldr r4, [r4, #12]
add r5, r8, r5
add r6, r9, r6
str r12, [r3]
add r7, r7, r4
stmib r3, {r5, r6, r7}
bne .LBB4_4
.LBB4_5:
pop {r4, r5, r6, r7, r8, r9, r11, pc}
Instead of rerolling this loop to vectorize it into a load+load+add+store,
Clang will generate a 512-bit vector and interleave it with a number of
shuffles, as evident by the LLVM output:
define dso_local void @add_unroll_while_reverse(i32* noalias nocapture, i32*
noalias nocapture readonly) local_unnamed_addr #0 {
%3 = getelementptr i32, i32* %0, i32 1024
%4 = getelementptr i32, i32* %1, i32 1024
%5 = icmp ugt i32* %4, %0
%6 = icmp ugt i32* %3, %1
%7 = and i1 %5, %6
br i1 %7, label %34, label %8
; <label>:8: ; preds = %2, %8
%9 = phi i32 [ %32, %8 ], [ 0, %2 ]
%10 = shl i32 %9, 2
%11 = getelementptr i32, i32* %1, i32 %10
%12 = shl i32 %9, 2
%13 = getelementptr i32, i32* %0, i32 %12
%14 = bitcast i32* %11 to <16 x i32>*
%15 = load <16 x i32>, <16 x i32>* %14, align 4, !tbaa !5
%16 = bitcast i32* %13 to <16 x i32>*
%17 = load <16 x i32>, <16 x i32>* %16, align 4, !tbaa !5
%18 = add <16 x i32> %17, %15
%19 = shufflevector <16 x i32> %18, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12> %20 = add <16 x i32> %17, %15
%21 = shufflevector <16 x i32> %20, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
%22 = getelementptr inbounds i32, i32* %13, i32 3
%23 = add <16 x i32> %17, %15
%24 = shufflevector <16 x i32> %23, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
%25 = add <16 x i32> %17, %15
%26 = shufflevector <16 x i32> %25, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
%27 = getelementptr inbounds i32, i32* %22, i32 -3
%28 = bitcast i32* %27 to <16 x i32>*
%29 = shufflevector <4 x i32> %19, <4 x i32> %21, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%30 = shufflevector <4 x i32> %24, <4 x i32> %26, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%31 = shufflevector <8 x i32> %29, <8 x i32> %30, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
store <16 x i32> %31, <16 x i32>* %28, align 4, !tbaa !5
%32 = add i32 %9, 4
%33 = icmp eq i32 %32, 256
br i1 %33, label %48, label %8, !llvm.loop !13
; <label>:34: ; preds = %2, %34
%35 = phi i32 [ %46, %34 ], [ 1024, %2 ]
%36 = phi i32* [ %38, %34 ], [ %1, %2 ]
%37 = phi i32* [ %41, %34 ], [ %0, %2 ]
%38 = getelementptr inbounds i32, i32* %36, i32 4
%39 = bitcast i32* %36 to <4 x i32>*
%40 = load <4 x i32>, <4 x i32>* %39, align 4, !tbaa !5
%41 = getelementptr inbounds i32, i32* %37, i32 4
%42 = bitcast i32* %37 to <4 x i32>*
%43 = load <4 x i32>, <4 x i32>* %42, align 4, !tbaa !5
%44 = add <4 x i32> %43, %40
%45 = bitcast i32* %37 to <4 x i32>*
store <4 x i32> %44, <4 x i32>* %45, align 4, !tbaa !5
%46 = add nsw i32 %35, -4
%47 = icmp eq i32 %46, 0
br i1 %47, label %48, label %34, !llvm.loop !14
; <label>:48: ; preds = %8, %34
ret void
}
This is ARM NEON, there are no 512-bit vectors.