Autovectorization of a manually unrolled loop results in wide, interleaved vectors

[llvmbot]

Bugzilla Link	40490
Version	trunk
OS	All
Reporter	LLVM Bugzilla Contributor
CC	@hfinkel

[llvmbot]

Gah, stupid enter button kept submitting my changes before I wanted them to.

LLVM simply cannot vectorize an unrolled loop.

void add_unroll_while_reverse_ptradd(unsigned restrict p1, const unsigned restrict p2) { const unsigned const end = p1 + 1024; while (p1 != end) { p1++ += p2++; p1++ += p2++; p1++ += p2++; p1++ += *p2++; } }

Expected code output (ARM NEON, affects other platforms) would be something like this: add_unroll_while_reverse_ptradd: mov r2, #​0 .LBB0_1 add r3, r1, r2 vld1.32 {d16, d17}, [r3] add r3, r0, r2 add r2, r2, #​16 vld1.32 {d18, d19}, [r3] cmp r2, #​4096 vadd.i32 q8, q9, q8 vst1.32 {d16, d17}, [r3] bne .LBB0_1 bx lr

Actual output:

add_unroll_while_reverse_ptradd: push {r4, r5, r6, r7, r8, r9, r11, lr} add r2, r1, #​4096 cmp r2, r0 addhi r2, r0, #​4096 cmphi r2, r1 bhi .LBB4_3 mov lr, #​0 .LBB4_2: add r2, r1, lr add r12, r0, lr add lr, lr, #​64 mov r3, r2 mov r4, r12 vld1.32 {d16, d17}, [r3]! cmp lr, #​4096 vld1.32 {d18, d19}, [r4]! vadd.i32 q8, q9, q8 vld1.32 {d20, d21}, [r3] add r3, r2, #​48 add r2, r2, #​32 vld1.32 {d22, d23}, [r3] add r3, r12, #​48 vld1.32 {d26, d27}, [r2] add r2, r12, #​32 vld1.32 {d28, d29}, [r4] vadd.i32 q9, q14, q10 vld1.32 {d20, d21}, [r2] vadd.i32 q10, q10, q13 vld1.32 {d24, d25}, [r3] vorr q13, q8, q8 vadd.i32 q11, q12, q11 vorr q12, q9, q9 vorr q14, q10, q10 vorr q15, q11, q11 vtrn.32 q13, q12 vtrn.32 q14, q15 vorr q14, q10, q10 vext.32 q0, q12, q8, #​2 vzip.32 q14, q11 vzip.32 q8, q9 vext.32 q1, q10, q15, #​2 vext.32 q8, q10, q11, #​2 vext.32 q11, q15, q0, #​2 vext.32 q12, q1, q12, #​2 vext.32 q3, q11, q11, #​2 vext.32 q10, q10, q14, #​2 vext.32 q8, q8, q9, #​2 vext.32 q1, q12, q12, #​2 vext.32 q9, q10, q13, #​2 vext.32 q2, q8, q8, #​2 vext.32 q0, q9, q9, #​2 vst4.32 {d0, d2, d4, d6}, [r12]! vst4.32 {d1, d3, d5, d7}, [r12] bne .LBB4_2 b .LBB4_5 .LBB4_3: mov r2, #​0 .LBB4_4: mov r3, r0 mov r4, r1 ldr r12, [r3, r2, lsl #​2]! ldr lr, [r4, r2, lsl #​2]! add r2, r2, #​4 ldmib r3, {r8, r9} cmp r2, #​1024 add r12, r12, lr ldr r7, [r3, #​12] ldmib r4, {r5, r6} ldr r4, [r4, #​12] add r5, r8, r5 add r6, r9, r6 str r12, [r3] add r7, r7, r4 stmib r3, {r5, r6, r7} bne .LBB4_4 .LBB4_5: pop {r4, r5, r6, r7, r8, r9, r11, pc}

Instead of rerolling this loop to vectorize it into a load+load+add+store, Clang will generate a 512-bit vector and interleave it with a number of shuffles, as evident by the LLVM output:

define dso_local void @​add_unroll_while_reverse(i32 noalias nocapture, i32 noalias nocapture readonly) local_unnamed_addr #​0 { %3 = getelementptr i32, i32 %0, i32 1024 %4 = getelementptr i32, i32 %1, i32 1024 %5 = icmp ugt i32 %4, %0 %6 = icmp ugt i32 %3, %1 %7 = and i1 %5, %6 br i1 %7, label %34, label %8

; :8: ; preds = %2, %8 %9 = phi i32 [ %32, %8 ], [ 0, %2 ] %10 = shl i32 %9, 2 %11 = getelementptr i32, i32 %1, i32 %10 %12 = shl i32 %9, 2 %13 = getelementptr i32, i32 %0, i32 %12 %14 = bitcast i32 %11 to <16 x i32> %15 = load <16 x i32>, <16 x i32> %14, align 4, !tbaa !​5 %16 = bitcast i32 %13 to <16 x i32> %17 = load <16 x i32>, <16 x i32> %16, align 4, !tbaa !​5 %18 = add <16 x i32> %17, %15 %19 = shufflevector <16 x i32> %18, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12> %20 = add <16 x i32> %17, %15 %21 = shufflevector <16 x i32> %20, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13> %22 = getelementptr inbounds i32, i32 %13, i32 3 %23 = add <16 x i32> %17, %15 %24 = shufflevector <16 x i32> %23, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14> %25 = add <16 x i32> %17, %15 %26 = shufflevector <16 x i32> %25, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15> %27 = getelementptr inbounds i32, i32 %22, i32 -3 %28 = bitcast i32 %27 to <16 x i32> %29 = shufflevector <4 x i32> %19, <4 x i32> %21, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %30 = shufflevector <4 x i32> %24, <4 x i32> %26, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %31 = shufflevector <8 x i32> %29, <8 x i32> %30, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15> store <16 x i32> %31, <16 x i32>* %28, align 4, !tbaa !​5 %32 = add i32 %9, 4 %33 = icmp eq i32 %32, 256 br i1 %33, label %48, label %8, !llvm.loop !​13

; :34: ; preds = %2, %34 %35 = phi i32 [ %46, %34 ], [ 1024, %2 ] %36 = phi i32 [ %38, %34 ], [ %1, %2 ] %37 = phi i32 [ %41, %34 ], [ %0, %2 ] %38 = getelementptr inbounds i32, i32 %36, i32 4 %39 = bitcast i32 %36 to <4 x i32> %40 = load <4 x i32>, <4 x i32> %39, align 4, !tbaa !​5 %41 = getelementptr inbounds i32, i32 %37, i32 4 %42 = bitcast i32 %37 to <4 x i32> %43 = load <4 x i32>, <4 x i32> %42, align 4, !tbaa !​5 %44 = add <4 x i32> %43, %40 %45 = bitcast i32 %37 to <4 x i32> store <4 x i32> %44, <4 x i32>* %45, align 4, !tbaa !​5 %46 = add nsw i32 %35, -4 %47 = icmp eq i32 %46, 0 br i1 %47, label %48, label %34, !llvm.loop !​14

; :48: ; preds = %8, %34 ret void }

This is ARM NEON, there are no 512-bit vectors.