aco: fix vgpr alloc granule with wave32

We still need to increase the number of physical vgprs

Totals from affected shaders:
SGPRS: 671976 -> 675288 (0.49 %)
VGPRS: 550112 -> 562596 (2.27 %)
Spilled SGPRs: 0 -> 0 (0.00 %)
Spilled VGPRs: 0 -> 0 (0.00 %)
Code Size: 27621660 -> 27606532 (-0.05 %) bytes
Max Waves: 81083 -> 87833 (8.32 %)
Instructions: 5391560 -> 5389031 (-0.05 %)

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Daniel Schürmann <daniel@schuermann.dev>

1 files changed, 15 insertions, 3 deletions

diff --git a/src/amd/compiler/aco_live_var_analysis.cpp b/src/amd/compiler/aco_live_var_analysis.cpp
index 4255d56173b..2841ba208f6 100644
--- a/src/amd/compiler/aco_live_var_analysis.cpp
+++ b/src/amd/compiler/aco_live_var_analysis.cpp
@@ -264,6 +264,13 @@ uint16_t get_sgpr_alloc(Program *program, uint16_t addressable_sgprs)
    return align(std::max(sgprs, granule), granule);
 }
 
+uint16_t get_vgpr_alloc(Program *program, uint16_t addressable_vgprs)
+{
+   assert(addressable_vgprs <= program->vgpr_limit);
+   uint16_t granule = program->vgpr_alloc_granule + 1;
+   return align(std::max(addressable_vgprs, granule), granule);
+}
+
 uint16_t get_addr_sgpr_from_waves(Program *program, uint16_t max_waves)
 {
     uint16_t sgprs = program->physical_sgprs / max_waves & ~program->sgpr_alloc_granule;
@@ -271,6 +278,12 @@ uint16_t get_addr_sgpr_from_waves(Program *program, uint16_t max_waves)
     return std::min(sgprs, program->sgpr_limit);
 }
 
+uint16_t get_addr_vgpr_from_waves(Program *program, uint16_t max_waves)
+{
+    uint16_t vgprs = 256 / max_waves & ~program->vgpr_alloc_granule;
+    return std::min(vgprs, program->vgpr_limit);
+}
+
 void update_vgpr_sgpr_demand(Program* program, const RegisterDemand new_demand)
 {
    /* TODO: max_waves_per_simd, simd_per_cu and the number of physical vgprs for Navi */
@@ -281,14 +294,13 @@ void update_vgpr_sgpr_demand(Program* program, const RegisterDemand new_demand)
    unsigned simd_per_cu_wgp = wgp ? simd_per_cu * 2 : simd_per_cu;
    unsigned lds_limit = wgp ? program->lds_limit * 2 : program->lds_limit;
 
-   const int16_t vgpr_alloc = std::max<int16_t>(4, (new_demand.vgpr + 3) & ~3);
    /* this won't compile, register pressure reduction necessary */
    if (new_demand.vgpr > program->vgpr_limit || new_demand.sgpr > program->sgpr_limit) {
       program->num_waves = 0;
       program->max_reg_demand = new_demand;
    } else {
       program->num_waves = program->physical_sgprs / get_sgpr_alloc(program, new_demand.sgpr);
-      program->num_waves = std::min<uint16_t>(program->num_waves, 256 / vgpr_alloc);
+      program->num_waves = std::min<uint16_t>(program->num_waves, 256 / get_vgpr_alloc(program, new_demand.vgpr));
       program->max_waves = max_waves_per_simd;
 
       /* adjust max_waves for workgroup and LDS limits */
@@ -314,7 +326,7 @@ void update_vgpr_sgpr_demand(Program* program, const RegisterDemand new_demand)
 
       /* incorporate max_waves and calculate max_reg_demand */
       program->num_waves = std::min<uint16_t>(program->num_waves, program->max_waves);
-      program->max_reg_demand.vgpr = int16_t((256 / program->num_waves) & ~3);
+      program->max_reg_demand.vgpr = get_addr_vgpr_from_waves(program, program->num_waves);
       program->max_reg_demand.sgpr = get_addr_sgpr_from_waves(program, program->num_waves);
    }
 }