1//===- LowerVectorIntrinsics.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
10#include "llvm/IR/IRBuilder.h"
11#include "llvm/IR/Module.h"
12#include "llvm/IR/ProfDataUtils.h"
13
14#define DEBUG_TYPE "lower-vector-intrinsics"
15
16using namespace llvm;
17
18bool llvm::lowerUnaryVectorIntrinsicAsLoop(Module &M, CallInst *CI) {
19 Type *RetTy = CI->getType();
20 auto *StructRetTy = dyn_cast<StructType>(Val: RetTy);
21 unsigned NumResults = StructRetTy ? StructRetTy->getNumElements() : 1;
22 auto *VecTy =
23 cast<VectorType>(Val: StructRetTy ? StructRetTy->getElementType(N: 0) : RetTy);
24
25 BasicBlock *PreLoopBB = CI->getParent();
26 BasicBlock *PostLoopBB = nullptr;
27 Function *ParentFunc = PreLoopBB->getParent();
28 LLVMContext &Ctx = PreLoopBB->getContext();
29 Type *IdxTy = M.getDataLayout().getIndexType(C&: Ctx, AddressSpace: 0);
30
31 PostLoopBB = PreLoopBB->splitBasicBlock(I: CI);
32 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: ParentFunc, InsertBefore: PostLoopBB);
33 PreLoopBB->getTerminator()->setSuccessor(Idx: 0, BB: LoopBB);
34
35 // Loop preheader
36 IRBuilder<> PreLoopBuilder(PreLoopBB->getTerminator());
37 Value *LoopEnd =
38 PreLoopBuilder.CreateElementCount(Ty: IdxTy, EC: VecTy->getElementCount());
39
40 // Loop body
41 IRBuilder<> LoopBuilder(LoopBB);
42
43 PHINode *LoopIndex = LoopBuilder.CreatePHI(Ty: IdxTy, NumReservedValues: 2);
44 LoopIndex->addIncoming(V: ConstantInt::get(Ty: IdxTy, V: 0U), BB: PreLoopBB);
45
46 SmallVector<PHINode *, 2> ResultPhis(NumResults);
47 for (unsigned I = 0; I != NumResults; ++I) {
48 ResultPhis[I] = LoopBuilder.CreatePHI(Ty: VecTy, NumReservedValues: 2);
49 ResultPhis[I]->addIncoming(V: PoisonValue::get(T: VecTy), BB: PreLoopBB);
50 }
51
52 Value *Elem =
53 LoopBuilder.CreateExtractElement(Vec: CI->getArgOperand(i: 0), Idx: LoopIndex);
54 Function *Fn = Intrinsic::getOrInsertDeclaration(M: &M, id: CI->getIntrinsicID(),
55 OverloadTys: VecTy->getElementType());
56
57 CallInst *ScalarCall = LoopBuilder.CreateCall(Callee: Fn, Args: Elem);
58 if (isa<FPMathOperator>(Val: CI))
59 ScalarCall->copyFastMathFlags(I: CI);
60
61 SmallVector<Value *, 2> NewVecs(NumResults);
62 for (unsigned I = 0; I != NumResults; ++I) {
63 Value *ScalarRes = ScalarCall;
64 if (StructRetTy)
65 ScalarRes = LoopBuilder.CreateExtractValue(Agg: ScalarCall, Idxs: I);
66 NewVecs[I] =
67 LoopBuilder.CreateInsertElement(Vec: ResultPhis[I], NewElt: ScalarRes, Idx: LoopIndex);
68 ResultPhis[I]->addIncoming(V: NewVecs[I], BB: LoopBB);
69 }
70
71 Value *One = ConstantInt::get(Ty: IdxTy, V: 1U);
72 Value *NextLoopIndex = LoopBuilder.CreateAdd(LHS: LoopIndex, RHS: One);
73 LoopIndex->addIncoming(V: NextLoopIndex, BB: LoopBB);
74
75 Value *ExitCond =
76 LoopBuilder.CreateICmp(P: CmpInst::ICMP_EQ, LHS: NextLoopIndex, RHS: LoopEnd);
77 CondBrInst *Br = LoopBuilder.CreateCondBr(Cond: ExitCond, True: PostLoopBB, False: LoopBB);
78 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *Br, DEBUG_TYPE);
79
80 Value *Res = NewVecs[0];
81 if (StructRetTy) {
82 IRBuilder<> PostLoopBuilder(CI);
83 Res = PoisonValue::get(T: RetTy);
84 for (unsigned I = 0; I != NumResults; ++I)
85 Res = PostLoopBuilder.CreateInsertValue(Agg: Res, Val: NewVecs[I], Idxs: I);
86 }
87
88 CI->replaceAllUsesWith(V: Res);
89 CI->eraseFromParent();
90 return true;
91}
92