1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
//===- AMDGPUReleaseVGPRs.cpp - Automatically release vgprs on GFX11+ -----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Insert S_SENDMSG instructions to release vgprs on GFX11+.
//
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
#include "AMDGPUSubtarget.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIDefines.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineOperand.h"
#include <optional>
using namespace llvm;
#define DEBUG_TYPE "release-vgprs"
namespace {
class AMDGPUReleaseVGPRs : public MachineFunctionPass {
public:
static char ID;
AMDGPUReleaseVGPRs() : MachineFunctionPass(ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
// Track if the last instruction referencing a vgpr in a MBB is a VMEM
// store. Because this pass is late in the pipeline, it is expected that the
// last vgpr use will likely be one of vmem store, ds, exp.
// Loads and others vgpr operations would have been
// deleted by this point, except for complex control flow involving loops.
// This is why we are just testing the type of instructions rather
// than the operands.
class LastVGPRUseIsVMEMStore {
BitVector BlockVMEMStore;
static std::optional<bool>
lastVGPRUseIsStore(const MachineBasicBlock &MBB) {
for (auto &MI : reverse(MBB.instrs())) {
// If it's a VMEM store, a VGPR will be used, return true.
if ((SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI)) &&
MI.mayStore())
return true;
// If it's referencing a VGPR but is not a VMEM store, return false.
if (SIInstrInfo::isDS(MI) || SIInstrInfo::isEXP(MI) ||
SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI) ||
SIInstrInfo::isVALU(MI))
return false;
}
// Wait until the values are propagated from the predecessors
return std::nullopt;
}
public:
LastVGPRUseIsVMEMStore(const MachineFunction &MF)
: BlockVMEMStore(MF.getNumBlockIDs()) {
df_iterator_default_set<const MachineBasicBlock *> Visited;
SmallVector<const MachineBasicBlock *> EndWithVMEMStoreBlocks;
for (const auto &MBB : MF) {
auto LastUseIsStore = lastVGPRUseIsStore(MBB);
if (!LastUseIsStore.has_value())
continue;
if (*LastUseIsStore) {
EndWithVMEMStoreBlocks.push_back(&MBB);
} else {
Visited.insert(&MBB);
}
}
for (const auto *MBB : EndWithVMEMStoreBlocks) {
for (const auto *Succ : depth_first_ext(MBB, Visited)) {
BlockVMEMStore[Succ->getNumber()] = true;
}
}
}
// Return true if the last instruction referencing a vgpr in this MBB
// is a VMEM store, otherwise return false.
bool isLastVGPRUseVMEMStore(const MachineBasicBlock &MBB) const {
return BlockVMEMStore[MBB.getNumber()];
}
};
static bool
runOnMachineBasicBlock(MachineBasicBlock &MBB, const SIInstrInfo *SII,
const LastVGPRUseIsVMEMStore &BlockVMEMStore) {
bool Changed = false;
for (auto &MI : MBB.terminators()) {
// Look for S_ENDPGM instructions
if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
// If the last instruction using a VGPR in the block is a VMEM store,
// release VGPRs. The VGPRs release will be placed just before ending
// the program
if (BlockVMEMStore.isLastVGPRUseVMEMStore(MBB)) {
BuildMI(MBB, MI, DebugLoc(), SII->get(AMDGPU::S_SENDMSG))
.addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
Changed = true;
}
}
}
return Changed;
}
bool runOnMachineFunction(MachineFunction &MF) override {
Function &F = MF.getFunction();
if (skipFunction(F) || !AMDGPU::isEntryFunctionCC(F.getCallingConv()))
return false;
// This pass only runs on GFX11+
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (ST.getGeneration() < AMDGPUSubtarget::GFX11)
return false;
LLVM_DEBUG(dbgs() << "AMDGPUReleaseVGPRs running on " << MF.getName()
<< "\n");
const SIInstrInfo *SII = ST.getInstrInfo();
LastVGPRUseIsVMEMStore BlockVMEMStore(MF);
bool Changed = false;
for (auto &MBB : MF) {
Changed |= runOnMachineBasicBlock(MBB, SII, BlockVMEMStore);
}
return Changed;
}
};
} // namespace
char AMDGPUReleaseVGPRs::ID = 0;
char &llvm::AMDGPUReleaseVGPRsID = AMDGPUReleaseVGPRs::ID;
INITIALIZE_PASS(AMDGPUReleaseVGPRs, DEBUG_TYPE, "Release VGPRs", false, false)
|
