1//===- LowerVectorIntrinsics.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
10#include "llvm/IR/IRBuilder.h"
11#include "llvm/IR/Module.h"
12
13#define DEBUG_TYPE "lower-vector-intrinsics"
14
15using namespace llvm;
16
17bool llvm::lowerUnaryVectorIntrinsicAsLoop(Module &M, CallInst *CI) {
18 Type *RetTy = CI->getType();
19 auto *StructRetTy = dyn_cast<StructType>(Val: RetTy);
20 unsigned NumResults = StructRetTy ? StructRetTy->getNumElements() : 1;
21 auto *VecTy =
22 cast<VectorType>(Val: StructRetTy ? StructRetTy->getElementType(N: 0) : RetTy);
23
24 BasicBlock *PreLoopBB = CI->getParent();
25 BasicBlock *PostLoopBB = nullptr;
26 Function *ParentFunc = PreLoopBB->getParent();
27 LLVMContext &Ctx = PreLoopBB->getContext();
28 Type *IdxTy = M.getDataLayout().getIndexType(C&: Ctx, AddressSpace: 0);
29
30 PostLoopBB = PreLoopBB->splitBasicBlock(I: CI);
31 BasicBlock *LoopBB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: ParentFunc, InsertBefore: PostLoopBB);
32 PreLoopBB->getTerminator()->setSuccessor(Idx: 0, BB: LoopBB);
33
34 // Loop preheader
35 IRBuilder<> PreLoopBuilder(PreLoopBB->getTerminator());
36 Value *LoopEnd =
37 PreLoopBuilder.CreateElementCount(Ty: IdxTy, EC: VecTy->getElementCount());
38
39 // Loop body
40 IRBuilder<> LoopBuilder(LoopBB);
41
42 PHINode *LoopIndex = LoopBuilder.CreatePHI(Ty: IdxTy, NumReservedValues: 2);
43 LoopIndex->addIncoming(V: ConstantInt::get(Ty: IdxTy, V: 0U), BB: PreLoopBB);
44
45 SmallVector<PHINode *, 2> ResultPhis(NumResults);
46 for (unsigned I = 0; I != NumResults; ++I) {
47 ResultPhis[I] = LoopBuilder.CreatePHI(Ty: VecTy, NumReservedValues: 2);
48 ResultPhis[I]->addIncoming(V: PoisonValue::get(T: VecTy), BB: PreLoopBB);
49 }
50
51 Value *Elem =
52 LoopBuilder.CreateExtractElement(Vec: CI->getArgOperand(i: 0), Idx: LoopIndex);
53 Function *Fn = Intrinsic::getOrInsertDeclaration(M: &M, id: CI->getIntrinsicID(),
54 OverloadTys: VecTy->getElementType());
55
56 CallInst *ScalarCall = LoopBuilder.CreateCall(Callee: Fn, Args: Elem);
57 if (isa<FPMathOperator>(Val: CI))
58 ScalarCall->copyFastMathFlags(I: CI);
59
60 SmallVector<Value *, 2> NewVecs(NumResults);
61 for (unsigned I = 0; I != NumResults; ++I) {
62 Value *ScalarRes = ScalarCall;
63 if (StructRetTy)
64 ScalarRes = LoopBuilder.CreateExtractValue(Agg: ScalarCall, Idxs: I);
65 NewVecs[I] =
66 LoopBuilder.CreateInsertElement(Vec: ResultPhis[I], NewElt: ScalarRes, Idx: LoopIndex);
67 ResultPhis[I]->addIncoming(V: NewVecs[I], BB: LoopBB);
68 }
69
70 Value *One = ConstantInt::get(Ty: IdxTy, V: 1U);
71 Value *NextLoopIndex = LoopBuilder.CreateAdd(LHS: LoopIndex, RHS: One);
72 LoopIndex->addIncoming(V: NextLoopIndex, BB: LoopBB);
73
74 Value *ExitCond =
75 LoopBuilder.CreateICmp(P: CmpInst::ICMP_EQ, LHS: NextLoopIndex, RHS: LoopEnd);
76 LoopBuilder.CreateCondBr(Cond: ExitCond, True: PostLoopBB, False: LoopBB);
77
78 Value *Res = NewVecs[0];
79 if (StructRetTy) {
80 IRBuilder<> PostLoopBuilder(CI);
81 Res = PoisonValue::get(T: RetTy);
82 for (unsigned I = 0; I != NumResults; ++I)
83 Res = PostLoopBuilder.CreateInsertValue(Agg: Res, Val: NewVecs[I], Idxs: I);
84 }
85
86 CI->replaceAllUsesWith(V: Res);
87 CI->eraseFromParent();
88 return true;
89}
90