blob: b723433c161cdbfd27e3e7039a0c8c71c7bf445c [file] [log] [blame]
Tom Stellard6b7d99d2012-12-19 22:10:31 +00001//===-- AMDGPUStructurizeCFG.cpp - ------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// The pass implemented in this file transforms the programs control flow
12/// graph into a form that's suitable for code generation on hardware that
13/// implements control flow by execution masking. This currently includes all
14/// AMD GPUs but may as well be useful for other types of hardware.
15//
16//===----------------------------------------------------------------------===//
17
18#include "AMDGPU.h"
Tom Stellard6b7d99d2012-12-19 22:10:31 +000019#include "llvm/ADT/SCCIterator.h"
Tom Stellard6b7d99d2012-12-19 22:10:31 +000020#include "llvm/Analysis/RegionInfo.h"
Chandler Carruth58a2cbe2013-01-02 10:22:59 +000021#include "llvm/Analysis/RegionIterator.h"
Tom Stellard6b7d99d2012-12-19 22:10:31 +000022#include "llvm/Analysis/RegionPass.h"
Chandler Carruth0b8c9a82013-01-02 11:36:10 +000023#include "llvm/IR/Module.h"
Tom Stellard6b7d99d2012-12-19 22:10:31 +000024#include "llvm/Transforms/Utils/SSAUpdater.h"
Christian Konigef6b2482013-02-16 11:27:50 +000025#include "llvm/Support/PatternMatch.h"
Tom Stellard6b7d99d2012-12-19 22:10:31 +000026
27using namespace llvm;
Christian Konigef6b2482013-02-16 11:27:50 +000028using namespace llvm::PatternMatch;
Tom Stellard6b7d99d2012-12-19 22:10:31 +000029
30namespace {
31
32// Definition of the complex types used in this pass.
33
34typedef std::pair<BasicBlock *, Value *> BBValuePair;
Tom Stellard6b7d99d2012-12-19 22:10:31 +000035
36typedef SmallVector<RegionNode*, 8> RNVector;
37typedef SmallVector<BasicBlock*, 8> BBVector;
Tom Stellard27f5d062013-02-08 22:24:37 +000038typedef SmallVector<BranchInst*, 8> BranchVector;
Tom Stellard6b7d99d2012-12-19 22:10:31 +000039typedef SmallVector<BBValuePair, 2> BBValueVector;
40
Tom Stellard27f5d062013-02-08 22:24:37 +000041typedef SmallPtrSet<BasicBlock *, 8> BBSet;
42
Tom Stellard6b7d99d2012-12-19 22:10:31 +000043typedef DenseMap<PHINode *, BBValueVector> PhiMap;
Christian Konigf0e469b2013-02-16 11:27:29 +000044typedef DenseMap<DomTreeNode *, unsigned> DTN2UnsignedMap;
Tom Stellard6b7d99d2012-12-19 22:10:31 +000045typedef DenseMap<BasicBlock *, PhiMap> BBPhiMap;
46typedef DenseMap<BasicBlock *, Value *> BBPredicates;
47typedef DenseMap<BasicBlock *, BBPredicates> PredMap;
Christian Konig623977d2013-02-16 11:27:45 +000048typedef DenseMap<BasicBlock *, BasicBlock*> BB2BBMap;
Tom Stellard13cf6cb2013-02-08 22:24:35 +000049typedef DenseMap<BasicBlock *, BBVector> BB2BBVecMap;
Tom Stellard6b7d99d2012-12-19 22:10:31 +000050
51// The name for newly created blocks.
52
53static const char *FlowBlockName = "Flow";
54
Christian Konigf0e469b2013-02-16 11:27:29 +000055/// @brief Find the nearest common dominator for multiple BasicBlocks
56///
57/// Helper class for AMDGPUStructurizeCFG
58/// TODO: Maybe move into common code
59class NearestCommonDominator {
60
61 DominatorTree *DT;
62
63 DTN2UnsignedMap IndexMap;
64
65 BasicBlock *Result;
66 unsigned ResultIndex;
67 bool ExplicitMentioned;
68
69public:
70 /// \brief Start a new query
71 NearestCommonDominator(DominatorTree *DomTree) {
72 DT = DomTree;
73 Result = 0;
74 }
75
76 /// \brief Add BB to the resulting dominator
77 void addBlock(BasicBlock *BB, bool Remember = true) {
78
79 DomTreeNode *Node = DT->getNode(BB);
80
81 if (Result == 0) {
82 unsigned Numbering = 0;
83 for (;Node;Node = Node->getIDom())
84 IndexMap[Node] = ++Numbering;
85 Result = BB;
86 ResultIndex = 1;
87 ExplicitMentioned = Remember;
88 return;
89 }
90
91 for (;Node;Node = Node->getIDom())
92 if (IndexMap.count(Node))
93 break;
94 else
95 IndexMap[Node] = 0;
96
97 assert(Node && "Dominator tree invalid!");
98
99 unsigned Numbering = IndexMap[Node];
100 if (Numbering > ResultIndex) {
101 Result = Node->getBlock();
102 ResultIndex = Numbering;
103 ExplicitMentioned = Remember && (Result == BB);
104 } else if (Numbering == ResultIndex) {
105 ExplicitMentioned |= Remember;
106 }
107 }
108
109 /// \brief Is "Result" one of the BBs added with "Remember" = True?
110 bool wasResultExplicitMentioned() {
111 return ExplicitMentioned;
112 }
113
114 /// \brief Get the query result
115 BasicBlock *getResult() {
116 return Result;
117 }
118};
119
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000120/// @brief Transforms the control flow graph on one single entry/exit region
121/// at a time.
122///
123/// After the transform all "If"/"Then"/"Else" style control flow looks like
124/// this:
125///
126/// \verbatim
127/// 1
128/// ||
129/// | |
130/// 2 |
131/// | /
132/// |/
133/// 3
134/// || Where:
135/// | | 1 = "If" block, calculates the condition
136/// 4 | 2 = "Then" subregion, runs if the condition is true
137/// | / 3 = "Flow" blocks, newly inserted flow blocks, rejoins the flow
138/// |/ 4 = "Else" optional subregion, runs if the condition is false
139/// 5 5 = "End" block, also rejoins the control flow
140/// \endverbatim
141///
142/// Control flow is expressed as a branch where the true exit goes into the
143/// "Then"/"Else" region, while the false exit skips the region
144/// The condition for the optional "Else" region is expressed as a PHI node.
145/// The incomming values of the PHI node are true for the "If" edge and false
146/// for the "Then" edge.
147///
148/// Additionally to that even complicated loops look like this:
149///
150/// \verbatim
151/// 1
152/// ||
153/// | |
154/// 2 ^ Where:
155/// | / 1 = "Entry" block
156/// |/ 2 = "Loop" optional subregion, with all exits at "Flow" block
157/// 3 3 = "Flow" block, with back edge to entry block
158/// |
159/// \endverbatim
160///
161/// The back edge of the "Flow" block is always on the false side of the branch
162/// while the true side continues the general flow. So the loop condition
163/// consist of a network of PHI nodes where the true incoming values expresses
164/// breaks and the false values expresses continue states.
165class AMDGPUStructurizeCFG : public RegionPass {
166
167 static char ID;
168
169 Type *Boolean;
170 ConstantInt *BoolTrue;
171 ConstantInt *BoolFalse;
172 UndefValue *BoolUndef;
173
174 Function *Func;
175 Region *ParentRegion;
176
177 DominatorTree *DT;
178
179 RNVector Order;
Tom Stellardf4e471a2013-02-08 22:24:38 +0000180 BBSet Visited;
Christian Konig623977d2013-02-16 11:27:45 +0000181
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000182 BBPhiMap DeletedPhis;
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000183 BB2BBVecMap AddedPhis;
Christian Konig623977d2013-02-16 11:27:45 +0000184
185 PredMap Predicates;
Tom Stellard27f5d062013-02-08 22:24:37 +0000186 BranchVector Conditions;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000187
Christian Konig623977d2013-02-16 11:27:45 +0000188 BB2BBMap Loops;
189 PredMap LoopPreds;
190 BranchVector LoopConds;
191
192 RegionNode *PrevNode;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000193
194 void orderNodes();
195
Christian Konig623977d2013-02-16 11:27:45 +0000196 void analyzeLoops(RegionNode *N);
197
Christian Konigef6b2482013-02-16 11:27:50 +0000198 Value *invert(Value *Condition);
199
Tom Stellard27f5d062013-02-08 22:24:37 +0000200 Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000201
Christian Konig623977d2013-02-16 11:27:45 +0000202 void gatherPredicates(RegionNode *N);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000203
204 void collectInfos();
205
Christian Konig623977d2013-02-16 11:27:45 +0000206 void insertConditions(bool Loops);
Tom Stellard27f5d062013-02-08 22:24:37 +0000207
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000208 void delPhiValues(BasicBlock *From, BasicBlock *To);
209
210 void addPhiValues(BasicBlock *From, BasicBlock *To);
211
212 void setPhiValues();
213
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000214 void killTerminator(BasicBlock *BB);
215
Tom Stellardf4e471a2013-02-08 22:24:38 +0000216 void changeExit(RegionNode *Node, BasicBlock *NewExit,
217 bool IncludeDominator);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000218
Tom Stellardf4e471a2013-02-08 22:24:38 +0000219 BasicBlock *getNextFlow(BasicBlock *Dominator);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000220
Christian Konig623977d2013-02-16 11:27:45 +0000221 BasicBlock *needPrefix(bool NeedEmpty);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000222
Tom Stellardf4e471a2013-02-08 22:24:38 +0000223 BasicBlock *needPostfix(BasicBlock *Flow, bool ExitUseAllowed);
224
Christian Konig623977d2013-02-16 11:27:45 +0000225 void setPrevNode(BasicBlock *BB);
Tom Stellardf4e471a2013-02-08 22:24:38 +0000226
227 bool dominatesPredicates(BasicBlock *BB, RegionNode *Node);
228
Christian Konig623977d2013-02-16 11:27:45 +0000229 bool isPredictableTrue(RegionNode *Node);
Tom Stellardf4e471a2013-02-08 22:24:38 +0000230
Christian Konig623977d2013-02-16 11:27:45 +0000231 void wireFlow(bool ExitUseAllowed, BasicBlock *LoopEnd);
232
233 void handleLoops(bool ExitUseAllowed, BasicBlock *LoopEnd);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000234
235 void createFlow();
236
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000237 void rebuildSSA();
238
239public:
240 AMDGPUStructurizeCFG():
241 RegionPass(ID) {
242
243 initializeRegionInfoPass(*PassRegistry::getPassRegistry());
244 }
245
Christian Konig777962f2013-03-01 09:46:11 +0000246 using Pass::doInitialization;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000247 virtual bool doInitialization(Region *R, RGPassManager &RGM);
248
249 virtual bool runOnRegion(Region *R, RGPassManager &RGM);
250
251 virtual const char *getPassName() const {
252 return "AMDGPU simplify control flow";
253 }
254
255 void getAnalysisUsage(AnalysisUsage &AU) const {
256
257 AU.addRequired<DominatorTree>();
258 AU.addPreserved<DominatorTree>();
259 RegionPass::getAnalysisUsage(AU);
260 }
261
262};
263
264} // end anonymous namespace
265
266char AMDGPUStructurizeCFG::ID = 0;
267
268/// \brief Initialize the types and constants used in the pass
269bool AMDGPUStructurizeCFG::doInitialization(Region *R, RGPassManager &RGM) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000270 LLVMContext &Context = R->getEntry()->getContext();
271
272 Boolean = Type::getInt1Ty(Context);
273 BoolTrue = ConstantInt::getTrue(Context);
274 BoolFalse = ConstantInt::getFalse(Context);
275 BoolUndef = UndefValue::get(Boolean);
276
277 return false;
278}
279
280/// \brief Build up the general order of nodes
281void AMDGPUStructurizeCFG::orderNodes() {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000282 scc_iterator<Region *> I = scc_begin(ParentRegion),
283 E = scc_end(ParentRegion);
284 for (Order.clear(); I != E; ++I) {
285 std::vector<RegionNode *> &Nodes = *I;
286 Order.append(Nodes.begin(), Nodes.end());
287 }
288}
289
Christian Konig623977d2013-02-16 11:27:45 +0000290/// \brief Determine the end of the loops
291void AMDGPUStructurizeCFG::analyzeLoops(RegionNode *N) {
292
293 if (N->isSubRegion()) {
294 // Test for exit as back edge
295 BasicBlock *Exit = N->getNodeAs<Region>()->getExit();
296 if (Visited.count(Exit))
297 Loops[Exit] = N->getEntry();
298
299 } else {
300 // Test for sucessors as back edge
301 BasicBlock *BB = N->getNodeAs<BasicBlock>();
302 BranchInst *Term = cast<BranchInst>(BB->getTerminator());
303
304 for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
305 BasicBlock *Succ = Term->getSuccessor(i);
306
307 if (Visited.count(Succ))
308 Loops[Succ] = BB;
309 }
310 }
311}
312
Christian Konigef6b2482013-02-16 11:27:50 +0000313/// \brief Invert the given condition
314Value *AMDGPUStructurizeCFG::invert(Value *Condition) {
315
316 // First: Check if it's a constant
317 if (Condition == BoolTrue)
318 return BoolFalse;
319
320 if (Condition == BoolFalse)
321 return BoolTrue;
322
323 if (Condition == BoolUndef)
324 return BoolUndef;
325
326 // Second: If the condition is already inverted, return the original value
327 if (match(Condition, m_Not(m_Value(Condition))))
328 return Condition;
329
330 // Third: Check all the users for an invert
331 BasicBlock *Parent = cast<Instruction>(Condition)->getParent();
332 for (Value::use_iterator I = Condition->use_begin(),
333 E = Condition->use_end(); I != E; ++I) {
334
335 Instruction *User = dyn_cast<Instruction>(*I);
336 if (!User || User->getParent() != Parent)
337 continue;
338
339 if (match(*I, m_Not(m_Specific(Condition))))
340 return *I;
341 }
342
343 // Last option: Create a new instruction
344 return BinaryOperator::CreateNot(Condition, "", Parent->getTerminator());
345}
346
Tom Stellard27f5d062013-02-08 22:24:37 +0000347/// \brief Build the condition for one edge
348Value *AMDGPUStructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
349 bool Invert) {
350 Value *Cond = Invert ? BoolFalse : BoolTrue;
351 if (Term->isConditional()) {
352 Cond = Term->getCondition();
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000353
Tom Stellard27f5d062013-02-08 22:24:37 +0000354 if (Idx != Invert)
Christian Konigef6b2482013-02-16 11:27:50 +0000355 Cond = invert(Cond);
Tom Stellard27f5d062013-02-08 22:24:37 +0000356 }
357 return Cond;
358}
359
Tom Stellard27f5d062013-02-08 22:24:37 +0000360/// \brief Analyze the predecessors of each block and build up predicates
Christian Konig623977d2013-02-16 11:27:45 +0000361void AMDGPUStructurizeCFG::gatherPredicates(RegionNode *N) {
362
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000363 RegionInfo *RI = ParentRegion->getRegionInfo();
Tom Stellard27f5d062013-02-08 22:24:37 +0000364 BasicBlock *BB = N->getEntry();
365 BBPredicates &Pred = Predicates[BB];
Christian Konig623977d2013-02-16 11:27:45 +0000366 BBPredicates &LPred = LoopPreds[BB];
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000367
Tom Stellard27f5d062013-02-08 22:24:37 +0000368 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
369 PI != PE; ++PI) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000370
Christian Konig623977d2013-02-16 11:27:45 +0000371 // Ignore it if it's a branch from outside into our region entry
372 if (!ParentRegion->contains(*PI))
Tom Stellard27f5d062013-02-08 22:24:37 +0000373 continue;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000374
Tom Stellard27f5d062013-02-08 22:24:37 +0000375 Region *R = RI->getRegionFor(*PI);
376 if (R == ParentRegion) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000377
Tom Stellard27f5d062013-02-08 22:24:37 +0000378 // It's a top level block in our region
379 BranchInst *Term = cast<BranchInst>((*PI)->getTerminator());
380 for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
381 BasicBlock *Succ = Term->getSuccessor(i);
382 if (Succ != BB)
383 continue;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000384
Tom Stellard27f5d062013-02-08 22:24:37 +0000385 if (Visited.count(*PI)) {
386 // Normal forward edge
387 if (Term->isConditional()) {
388 // Try to treat it like an ELSE block
389 BasicBlock *Other = Term->getSuccessor(!i);
Christian Konig623977d2013-02-16 11:27:45 +0000390 if (Visited.count(Other) && !Loops.count(Other) &&
Tom Stellard27f5d062013-02-08 22:24:37 +0000391 !Pred.count(Other) && !Pred.count(*PI)) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000392
Tom Stellard27f5d062013-02-08 22:24:37 +0000393 Pred[Other] = BoolFalse;
394 Pred[*PI] = BoolTrue;
395 continue;
396 }
397 }
Christian Konig623977d2013-02-16 11:27:45 +0000398 Pred[*PI] = buildCondition(Term, i, false);
399
Tom Stellard27f5d062013-02-08 22:24:37 +0000400 } else {
401 // Back edge
Christian Konig623977d2013-02-16 11:27:45 +0000402 LPred[*PI] = buildCondition(Term, i, true);
Tom Stellard27f5d062013-02-08 22:24:37 +0000403 }
Tom Stellard27f5d062013-02-08 22:24:37 +0000404 }
405
406 } else {
407
408 // It's an exit from a sub region
409 while(R->getParent() != ParentRegion)
410 R = R->getParent();
411
412 // Edge from inside a subregion to its entry, ignore it
413 if (R == N)
414 continue;
415
416 BasicBlock *Entry = R->getEntry();
Christian Konig623977d2013-02-16 11:27:45 +0000417 if (Visited.count(Entry))
418 Pred[Entry] = BoolTrue;
419 else
420 LPred[Entry] = BoolFalse;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000421 }
422 }
423}
424
425/// \brief Collect various loop and predicate infos
426void AMDGPUStructurizeCFG::collectInfos() {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000427
428 // Reset predicate
429 Predicates.clear();
430
431 // and loop infos
Christian Konig623977d2013-02-16 11:27:45 +0000432 Loops.clear();
433 LoopPreds.clear();
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000434
Tom Stellard27f5d062013-02-08 22:24:37 +0000435 // Reset the visited nodes
436 Visited.clear();
437
438 for (RNVector::reverse_iterator OI = Order.rbegin(), OE = Order.rend();
439 OI != OE; ++OI) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000440
441 // Analyze all the conditions leading to a node
Christian Konig623977d2013-02-16 11:27:45 +0000442 gatherPredicates(*OI);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000443
Tom Stellard27f5d062013-02-08 22:24:37 +0000444 // Remember that we've seen this node
Tom Stellardf4e471a2013-02-08 22:24:38 +0000445 Visited.insert((*OI)->getEntry());
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000446
Christian Konig623977d2013-02-16 11:27:45 +0000447 // Find the last back edges
448 analyzeLoops(*OI);
Tom Stellard27f5d062013-02-08 22:24:37 +0000449 }
Tom Stellard27f5d062013-02-08 22:24:37 +0000450}
451
452/// \brief Insert the missing branch conditions
Christian Konig623977d2013-02-16 11:27:45 +0000453void AMDGPUStructurizeCFG::insertConditions(bool Loops) {
454 BranchVector &Conds = Loops ? LoopConds : Conditions;
455 Value *Default = Loops ? BoolTrue : BoolFalse;
Tom Stellard27f5d062013-02-08 22:24:37 +0000456 SSAUpdater PhiInserter;
457
Christian Konig623977d2013-02-16 11:27:45 +0000458 for (BranchVector::iterator I = Conds.begin(),
459 E = Conds.end(); I != E; ++I) {
Tom Stellard27f5d062013-02-08 22:24:37 +0000460
461 BranchInst *Term = *I;
Tom Stellard27f5d062013-02-08 22:24:37 +0000462 assert(Term->isConditional());
463
Christian Konig623977d2013-02-16 11:27:45 +0000464 BasicBlock *Parent = Term->getParent();
465 BasicBlock *SuccTrue = Term->getSuccessor(0);
466 BasicBlock *SuccFalse = Term->getSuccessor(1);
Tom Stellard27f5d062013-02-08 22:24:37 +0000467
Christian Konig25bd8842013-02-16 11:27:40 +0000468 PhiInserter.Initialize(Boolean, "");
469 PhiInserter.AddAvailableValue(&Func->getEntryBlock(), Default);
Christian Konig623977d2013-02-16 11:27:45 +0000470 PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default);
Christian Konig25bd8842013-02-16 11:27:40 +0000471
Christian Konig623977d2013-02-16 11:27:45 +0000472 BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue];
Christian Konig25bd8842013-02-16 11:27:40 +0000473
474 NearestCommonDominator Dominator(DT);
475 Dominator.addBlock(Parent, false);
476
477 Value *ParentValue = 0;
Tom Stellard27f5d062013-02-08 22:24:37 +0000478 for (BBPredicates::iterator PI = Preds.begin(), PE = Preds.end();
479 PI != PE; ++PI) {
480
Christian Konig25bd8842013-02-16 11:27:40 +0000481 if (PI->first == Parent) {
482 ParentValue = PI->second;
483 break;
484 }
Tom Stellard27f5d062013-02-08 22:24:37 +0000485 PhiInserter.AddAvailableValue(PI->first, PI->second);
Christian Konig25bd8842013-02-16 11:27:40 +0000486 Dominator.addBlock(PI->first);
Tom Stellard27f5d062013-02-08 22:24:37 +0000487 }
488
Christian Konig25bd8842013-02-16 11:27:40 +0000489 if (ParentValue) {
490 Term->setCondition(ParentValue);
491 } else {
492 if (!Dominator.wasResultExplicitMentioned())
493 PhiInserter.AddAvailableValue(Dominator.getResult(), Default);
494
Tom Stellard27f5d062013-02-08 22:24:37 +0000495 Term->setCondition(PhiInserter.GetValueInMiddleOfBlock(Parent));
Christian Konig25bd8842013-02-16 11:27:40 +0000496 }
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000497 }
498}
499
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000500/// \brief Remove all PHI values coming from "From" into "To" and remember
501/// them in DeletedPhis
502void AMDGPUStructurizeCFG::delPhiValues(BasicBlock *From, BasicBlock *To) {
503 PhiMap &Map = DeletedPhis[To];
504 for (BasicBlock::iterator I = To->begin(), E = To->end();
505 I != E && isa<PHINode>(*I);) {
506
507 PHINode &Phi = cast<PHINode>(*I++);
508 while (Phi.getBasicBlockIndex(From) != -1) {
509 Value *Deleted = Phi.removeIncomingValue(From, false);
510 Map[&Phi].push_back(std::make_pair(From, Deleted));
511 }
512 }
513}
514
515/// \brief Add a dummy PHI value as soon as we knew the new predecessor
516void AMDGPUStructurizeCFG::addPhiValues(BasicBlock *From, BasicBlock *To) {
517 for (BasicBlock::iterator I = To->begin(), E = To->end();
518 I != E && isa<PHINode>(*I);) {
519
520 PHINode &Phi = cast<PHINode>(*I++);
521 Value *Undef = UndefValue::get(Phi.getType());
522 Phi.addIncoming(Undef, From);
523 }
524 AddedPhis[To].push_back(From);
525}
526
527/// \brief Add the real PHI value as soon as everything is set up
528void AMDGPUStructurizeCFG::setPhiValues() {
529
530 SSAUpdater Updater;
531 for (BB2BBVecMap::iterator AI = AddedPhis.begin(), AE = AddedPhis.end();
532 AI != AE; ++AI) {
533
534 BasicBlock *To = AI->first;
535 BBVector &From = AI->second;
536
537 if (!DeletedPhis.count(To))
538 continue;
539
540 PhiMap &Map = DeletedPhis[To];
541 for (PhiMap::iterator PI = Map.begin(), PE = Map.end();
542 PI != PE; ++PI) {
543
544 PHINode *Phi = PI->first;
545 Value *Undef = UndefValue::get(Phi->getType());
546 Updater.Initialize(Phi->getType(), "");
547 Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
548 Updater.AddAvailableValue(To, Undef);
549
Christian Konig4c79c712013-02-16 11:27:35 +0000550 NearestCommonDominator Dominator(DT);
551 Dominator.addBlock(To, false);
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000552 for (BBValueVector::iterator VI = PI->second.begin(),
553 VE = PI->second.end(); VI != VE; ++VI) {
554
555 Updater.AddAvailableValue(VI->first, VI->second);
Christian Konig4c79c712013-02-16 11:27:35 +0000556 Dominator.addBlock(VI->first);
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000557 }
558
Christian Konig4c79c712013-02-16 11:27:35 +0000559 if (!Dominator.wasResultExplicitMentioned())
560 Updater.AddAvailableValue(Dominator.getResult(), Undef);
561
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000562 for (BBVector::iterator FI = From.begin(), FE = From.end();
563 FI != FE; ++FI) {
564
565 int Idx = Phi->getBasicBlockIndex(*FI);
566 assert(Idx != -1);
567 Phi->setIncomingValue(Idx, Updater.GetValueAtEndOfBlock(*FI));
568 }
569 }
570
571 DeletedPhis.erase(To);
572 }
573 assert(DeletedPhis.empty());
574}
575
Tom Stellardf4e471a2013-02-08 22:24:38 +0000576/// \brief Remove phi values from all successors and then remove the terminator.
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000577void AMDGPUStructurizeCFG::killTerminator(BasicBlock *BB) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000578 TerminatorInst *Term = BB->getTerminator();
579 if (!Term)
580 return;
581
582 for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
583 SI != SE; ++SI) {
584
585 delPhiValues(BB, *SI);
586 }
587
588 Term->eraseFromParent();
589}
590
Tom Stellardf4e471a2013-02-08 22:24:38 +0000591/// \brief Let node exit(s) point to NewExit
592void AMDGPUStructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
593 bool IncludeDominator) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000594
Tom Stellardf4e471a2013-02-08 22:24:38 +0000595 if (Node->isSubRegion()) {
596 Region *SubRegion = Node->getNodeAs<Region>();
597 BasicBlock *OldExit = SubRegion->getExit();
598 BasicBlock *Dominator = 0;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000599
Tom Stellardf4e471a2013-02-08 22:24:38 +0000600 // Find all the edges from the sub region to the exit
601 for (pred_iterator I = pred_begin(OldExit), E = pred_end(OldExit);
602 I != E;) {
603
604 BasicBlock *BB = *I++;
605 if (!SubRegion->contains(BB))
606 continue;
607
608 // Modify the edges to point to the new exit
609 delPhiValues(BB, OldExit);
610 BB->getTerminator()->replaceUsesOfWith(OldExit, NewExit);
611 addPhiValues(BB, NewExit);
612
613 // Find the new dominator (if requested)
614 if (IncludeDominator) {
615 if (!Dominator)
616 Dominator = BB;
617 else
618 Dominator = DT->findNearestCommonDominator(Dominator, BB);
619 }
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000620 }
621
Tom Stellardf4e471a2013-02-08 22:24:38 +0000622 // Change the dominator (if requested)
623 if (Dominator)
624 DT->changeImmediateDominator(NewExit, Dominator);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000625
Tom Stellardf4e471a2013-02-08 22:24:38 +0000626 // Update the region info
627 SubRegion->replaceExit(NewExit);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000628
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000629 } else {
Tom Stellardf4e471a2013-02-08 22:24:38 +0000630 BasicBlock *BB = Node->getNodeAs<BasicBlock>();
631 killTerminator(BB);
632 BranchInst::Create(NewExit, BB);
633 addPhiValues(BB, NewExit);
634 if (IncludeDominator)
635 DT->changeImmediateDominator(NewExit, BB);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000636 }
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000637}
638
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000639/// \brief Create a new flow node and update dominator tree and region info
Tom Stellardf4e471a2013-02-08 22:24:38 +0000640BasicBlock *AMDGPUStructurizeCFG::getNextFlow(BasicBlock *Dominator) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000641 LLVMContext &Context = Func->getContext();
642 BasicBlock *Insert = Order.empty() ? ParentRegion->getExit() :
643 Order.back()->getEntry();
644 BasicBlock *Flow = BasicBlock::Create(Context, FlowBlockName,
645 Func, Insert);
Tom Stellardf4e471a2013-02-08 22:24:38 +0000646 DT->addNewBlock(Flow, Dominator);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000647 ParentRegion->getRegionInfo()->setRegionFor(Flow, ParentRegion);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000648 return Flow;
649}
650
Tom Stellardf4e471a2013-02-08 22:24:38 +0000651/// \brief Create a new or reuse the previous node as flow node
Christian Konig623977d2013-02-16 11:27:45 +0000652BasicBlock *AMDGPUStructurizeCFG::needPrefix(bool NeedEmpty) {
Tom Stellardf4e471a2013-02-08 22:24:38 +0000653
Christian Konig623977d2013-02-16 11:27:45 +0000654 BasicBlock *Entry = PrevNode->getEntry();
Tom Stellardf4e471a2013-02-08 22:24:38 +0000655
Christian Konig623977d2013-02-16 11:27:45 +0000656 if (!PrevNode->isSubRegion()) {
657 killTerminator(Entry);
658 if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end())
659 return Entry;
Tom Stellardf4e471a2013-02-08 22:24:38 +0000660
Christian Konig623977d2013-02-16 11:27:45 +0000661 }
Tom Stellardf4e471a2013-02-08 22:24:38 +0000662
Christian Konig623977d2013-02-16 11:27:45 +0000663 // create a new flow node
664 BasicBlock *Flow = getNextFlow(Entry);
Tom Stellardf4e471a2013-02-08 22:24:38 +0000665
Christian Konig623977d2013-02-16 11:27:45 +0000666 // and wire it up
667 changeExit(PrevNode, Flow, true);
668 PrevNode = ParentRegion->getBBNode(Flow);
669 return Flow;
Tom Stellardf4e471a2013-02-08 22:24:38 +0000670}
671
672/// \brief Returns the region exit if possible, otherwise just a new flow node
673BasicBlock *AMDGPUStructurizeCFG::needPostfix(BasicBlock *Flow,
674 bool ExitUseAllowed) {
675
676 if (Order.empty() && ExitUseAllowed) {
677 BasicBlock *Exit = ParentRegion->getExit();
678 DT->changeImmediateDominator(Exit, Flow);
679 addPhiValues(Flow, Exit);
680 return Exit;
681 }
682 return getNextFlow(Flow);
683}
684
Christian Konig623977d2013-02-16 11:27:45 +0000685/// \brief Set the previous node
686void AMDGPUStructurizeCFG::setPrevNode(BasicBlock *BB) {
687 PrevNode = ParentRegion->contains(BB) ? ParentRegion->getBBNode(BB) : 0;
Tom Stellardf4e471a2013-02-08 22:24:38 +0000688}
689
690/// \brief Does BB dominate all the predicates of Node ?
691bool AMDGPUStructurizeCFG::dominatesPredicates(BasicBlock *BB, RegionNode *Node) {
692 BBPredicates &Preds = Predicates[Node->getEntry()];
693 for (BBPredicates::iterator PI = Preds.begin(), PE = Preds.end();
694 PI != PE; ++PI) {
695
696 if (!DT->dominates(BB, PI->first))
697 return false;
698 }
699 return true;
700}
701
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000702/// \brief Can we predict that this node will always be called?
Christian Konig623977d2013-02-16 11:27:45 +0000703bool AMDGPUStructurizeCFG::isPredictableTrue(RegionNode *Node) {
Tom Stellardf4e471a2013-02-08 22:24:38 +0000704
Christian Konig623977d2013-02-16 11:27:45 +0000705 BBPredicates &Preds = Predicates[Node->getEntry()];
706 bool Dominated = false;
707
708 // Regionentry is always true
709 if (PrevNode == 0)
710 return true;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000711
712 for (BBPredicates::iterator I = Preds.begin(), E = Preds.end();
713 I != E; ++I) {
714
715 if (I->second != BoolTrue)
716 return false;
717
Christian Konig623977d2013-02-16 11:27:45 +0000718 if (!Dominated && DT->dominates(I->first, PrevNode->getEntry()))
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000719 Dominated = true;
720 }
Tom Stellardf4e471a2013-02-08 22:24:38 +0000721
722 // TODO: The dominator check is too strict
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000723 return Dominated;
724}
725
Tom Stellardf4e471a2013-02-08 22:24:38 +0000726/// Take one node from the order vector and wire it up
Christian Konig623977d2013-02-16 11:27:45 +0000727void AMDGPUStructurizeCFG::wireFlow(bool ExitUseAllowed,
728 BasicBlock *LoopEnd) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000729
Tom Stellardf4e471a2013-02-08 22:24:38 +0000730 RegionNode *Node = Order.pop_back_val();
Christian Konig623977d2013-02-16 11:27:45 +0000731 Visited.insert(Node->getEntry());
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000732
Christian Konig623977d2013-02-16 11:27:45 +0000733 if (isPredictableTrue(Node)) {
Tom Stellardf4e471a2013-02-08 22:24:38 +0000734 // Just a linear flow
Christian Konig623977d2013-02-16 11:27:45 +0000735 if (PrevNode) {
736 changeExit(PrevNode, Node->getEntry(), true);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000737 }
Christian Konig623977d2013-02-16 11:27:45 +0000738 PrevNode = Node;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000739
740 } else {
Tom Stellardf4e471a2013-02-08 22:24:38 +0000741 // Insert extra prefix node (or reuse last one)
Christian Konig623977d2013-02-16 11:27:45 +0000742 BasicBlock *Flow = needPrefix(false);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000743
Tom Stellardf4e471a2013-02-08 22:24:38 +0000744 // Insert extra postfix node (or use exit instead)
745 BasicBlock *Entry = Node->getEntry();
Christian Konig623977d2013-02-16 11:27:45 +0000746 BasicBlock *Next = needPostfix(Flow, ExitUseAllowed);
Tom Stellardf4e471a2013-02-08 22:24:38 +0000747
748 // let it point to entry and next block
749 Conditions.push_back(BranchInst::Create(Entry, Next, BoolUndef, Flow));
750 addPhiValues(Flow, Entry);
751 DT->changeImmediateDominator(Entry, Flow);
752
Christian Konig623977d2013-02-16 11:27:45 +0000753 PrevNode = Node;
754 while (!Order.empty() && !Visited.count(LoopEnd) &&
Tom Stellardf4e471a2013-02-08 22:24:38 +0000755 dominatesPredicates(Entry, Order.back())) {
Christian Konig623977d2013-02-16 11:27:45 +0000756 handleLoops(false, LoopEnd);
Tom Stellardf4e471a2013-02-08 22:24:38 +0000757 }
758
Christian Konig623977d2013-02-16 11:27:45 +0000759 changeExit(PrevNode, Next, false);
760 setPrevNode(Next);
761 }
762}
763
764void AMDGPUStructurizeCFG::handleLoops(bool ExitUseAllowed,
765 BasicBlock *LoopEnd) {
766 RegionNode *Node = Order.back();
767 BasicBlock *LoopStart = Node->getEntry();
768
769 if (!Loops.count(LoopStart)) {
770 wireFlow(ExitUseAllowed, LoopEnd);
771 return;
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000772 }
773
Christian Konig623977d2013-02-16 11:27:45 +0000774 if (!isPredictableTrue(Node))
775 LoopStart = needPrefix(true);
776
777 LoopEnd = Loops[Node->getEntry()];
778 wireFlow(false, LoopEnd);
779 while (!Visited.count(LoopEnd)) {
780 handleLoops(false, LoopEnd);
781 }
782
783 // Create an extra loop end node
784 LoopEnd = needPrefix(false);
785 BasicBlock *Next = needPostfix(LoopEnd, ExitUseAllowed);
786 LoopConds.push_back(BranchInst::Create(Next, LoopStart,
787 BoolUndef, LoopEnd));
788 addPhiValues(LoopEnd, LoopStart);
789 setPrevNode(Next);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000790}
791
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000792/// After this function control flow looks like it should be, but
Tom Stellardf4e471a2013-02-08 22:24:38 +0000793/// branches and PHI nodes only have undefined conditions.
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000794void AMDGPUStructurizeCFG::createFlow() {
Tom Stellardf4e471a2013-02-08 22:24:38 +0000795
796 BasicBlock *Exit = ParentRegion->getExit();
797 bool EntryDominatesExit = DT->dominates(ParentRegion->getEntry(), Exit);
798
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000799 DeletedPhis.clear();
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000800 AddedPhis.clear();
Tom Stellardf4e471a2013-02-08 22:24:38 +0000801 Conditions.clear();
Christian Konig623977d2013-02-16 11:27:45 +0000802 LoopConds.clear();
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000803
Christian Konig623977d2013-02-16 11:27:45 +0000804 PrevNode = 0;
805 Visited.clear();
806
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000807 while (!Order.empty()) {
Christian Konig623977d2013-02-16 11:27:45 +0000808 handleLoops(EntryDominatesExit, 0);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000809 }
810
Christian Konig623977d2013-02-16 11:27:45 +0000811 if (PrevNode)
812 changeExit(PrevNode, Exit, EntryDominatesExit);
Tom Stellardf4e471a2013-02-08 22:24:38 +0000813 else
814 assert(EntryDominatesExit);
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000815}
816
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000817/// Handle a rare case where the disintegrated nodes instructions
818/// no longer dominate all their uses. Not sure if this is really nessasary
819void AMDGPUStructurizeCFG::rebuildSSA() {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000820 SSAUpdater Updater;
821 for (Region::block_iterator I = ParentRegion->block_begin(),
822 E = ParentRegion->block_end();
823 I != E; ++I) {
824
825 BasicBlock *BB = *I;
826 for (BasicBlock::iterator II = BB->begin(), IE = BB->end();
827 II != IE; ++II) {
828
829 bool Initialized = false;
830 for (Use *I = &II->use_begin().getUse(), *Next; I; I = Next) {
831
832 Next = I->getNext();
833
834 Instruction *User = cast<Instruction>(I->getUser());
835 if (User->getParent() == BB) {
836 continue;
837
838 } else if (PHINode *UserPN = dyn_cast<PHINode>(User)) {
839 if (UserPN->getIncomingBlock(*I) == BB)
840 continue;
841 }
842
843 if (DT->dominates(II, User))
844 continue;
845
846 if (!Initialized) {
847 Value *Undef = UndefValue::get(II->getType());
848 Updater.Initialize(II->getType(), "");
849 Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
850 Updater.AddAvailableValue(BB, II);
851 Initialized = true;
852 }
853 Updater.RewriteUseAfterInsertions(*I);
854 }
855 }
856 }
857}
858
859/// \brief Run the transformation for each region found
860bool AMDGPUStructurizeCFG::runOnRegion(Region *R, RGPassManager &RGM) {
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000861 if (R->isTopLevelRegion())
862 return false;
863
864 Func = R->getEntry()->getParent();
865 ParentRegion = R;
866
867 DT = &getAnalysis<DominatorTree>();
868
869 orderNodes();
870 collectInfos();
871 createFlow();
Christian Konig623977d2013-02-16 11:27:45 +0000872 insertConditions(false);
873 insertConditions(true);
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000874 setPhiValues();
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000875 rebuildSSA();
876
Tom Stellard27f5d062013-02-08 22:24:37 +0000877 // Cleanup
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000878 Order.clear();
879 Visited.clear();
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000880 DeletedPhis.clear();
Tom Stellard13cf6cb2013-02-08 22:24:35 +0000881 AddedPhis.clear();
Christian Konig623977d2013-02-16 11:27:45 +0000882 Predicates.clear();
Tom Stellard27f5d062013-02-08 22:24:37 +0000883 Conditions.clear();
Christian Konig623977d2013-02-16 11:27:45 +0000884 Loops.clear();
885 LoopPreds.clear();
886 LoopConds.clear();
Tom Stellard6b7d99d2012-12-19 22:10:31 +0000887
888 return true;
889}
890
891/// \brief Create the pass
892Pass *llvm::createAMDGPUStructurizeCFGPass() {
893 return new AMDGPUStructurizeCFG();
894}