aco: Initial commit of independent AMD compiler

ACO (short for AMD Compiler) is a new compiler backend with the goal to replace
LLVM for Radeon hardware for the RADV driver.

ACO currently supports only VS, PS and CS on VI and Vega.
There are some optimizations missing because of unmerged NIR changes
which may decrease performance.

Full commit history can be found at
https://github.com/daniel-schuermann/mesa/commits/backend

Co-authored-by: Daniel Schürmann <daniel@schuermann.dev>
Co-authored-by: Rhys Perry <pendingchaos02@gmail.com>
Co-authored-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
Co-authored-by: Connor Abbott <cwabbott0@gmail.com>
Co-authored-by: Michael Schellenberger Costa <mschellenbergercosta@googlemail.com>
Co-authored-by: Timur Kristóf <timur.kristof@gmail.com>

Acked-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Acked-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>

1 files changed, 1366 insertions, 0 deletions

diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp
new file mode 100644
index 00000000000..6c4c408e659
--- /dev/null
+++ b/src/amd/compiler/aco_instruction_selection_setup.cpp
@@ -0,0 +1,1366 @@
+/*
+ * Copyright © 2018 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <unordered_map>
+#include "aco_ir.h"
+#include "nir.h"
+#include "vulkan/radv_shader.h"
+#include "vulkan/radv_descriptor_set.h"
+#include "sid.h"
+#include "ac_exp_param.h"
+
+#include "util/u_math.h"
+
+#define MAX_INLINE_PUSH_CONSTS 8
+
+namespace aco {
+
+enum fs_input {
+   persp_sample_p1,
+   persp_sample_p2,
+   persp_center_p1,
+   persp_center_p2,
+   persp_centroid_p1,
+   persp_centroid_p2,
+   persp_pull_model,
+   linear_sample_p1,
+   linear_sample_p2,
+   linear_center_p1,
+   linear_center_p2,
+   linear_centroid_p1,
+   linear_centroid_p2,
+   line_stipple,
+   frag_pos_0,
+   frag_pos_1,
+   frag_pos_2,
+   frag_pos_3,
+   front_face,
+   ancillary,
+   sample_coverage,
+   fixed_pt,
+   max_inputs,
+};
+
+struct vs_output_state {
+   uint8_t mask[VARYING_SLOT_VAR31 + 1];
+   Temp outputs[VARYING_SLOT_VAR31 + 1][4];
+};
+
+struct isel_context {
+   struct radv_nir_compiler_options *options;
+   Program *program;
+   nir_shader *shader;
+   uint32_t constant_data_offset;
+   Block *block;
+   bool *divergent_vals;
+   std::unique_ptr<Temp[]> allocated;
+   std::unordered_map<unsigned, std::array<Temp,4>> allocated_vec;
+   Stage stage; /* Stage */
+   struct {
+      bool has_branch;
+      uint16_t loop_nest_depth = 0;
+      struct {
+         unsigned header_idx;
+         Block* exit;
+         bool has_divergent_continue = false;
+         bool has_divergent_branch = false;
+      } parent_loop;
+      struct {
+         bool is_divergent = false;
+      } parent_if;
+      bool exec_potentially_empty = false;
+   } cf_info;
+
+   /* scratch */
+   bool scratch_enabled = false;
+   Temp private_segment_buffer = Temp(0, s2); /* also the part of the scratch descriptor on compute */
+   Temp scratch_offset = Temp(0, s1);
+
+   /* inputs common for merged stages */
+   Temp merged_wave_info = Temp(0, s1);
+
+   /* FS inputs */
+   bool fs_vgpr_args[fs_input::max_inputs];
+   Temp fs_inputs[fs_input::max_inputs];
+   Temp prim_mask = Temp(0, s1);
+   Temp descriptor_sets[MAX_SETS];
+   Temp push_constants = Temp(0, s1);
+   Temp inline_push_consts[MAX_INLINE_PUSH_CONSTS];
+   unsigned num_inline_push_consts = 0;
+   unsigned base_inline_push_consts = 0;
+
+   /* VS inputs */
+   Temp vertex_buffers = Temp(0, s1);
+   Temp base_vertex = Temp(0, s1);
+   Temp start_instance = Temp(0, s1);
+   Temp draw_id = Temp(0, s1);
+   Temp view_index = Temp(0, s1);
+   Temp es2gs_offset = Temp(0, s1);
+   Temp vertex_id = Temp(0, v1);
+   Temp rel_auto_id = Temp(0, v1);
+   Temp instance_id = Temp(0, v1);
+   Temp vs_prim_id = Temp(0, v1);
+   bool needs_instance_id;
+
+   /* CS inputs */
+   Temp num_workgroups[3] = {Temp(0, s1), Temp(0, s1), Temp(0, s1)};
+   Temp workgroup_ids[3] = {Temp(0, s1), Temp(0, s1), Temp(0, s1)};
+   Temp tg_size = Temp(0, s1);
+   Temp local_invocation_ids[3] = {Temp(0, v1), Temp(0, v1), Temp(0, v1)};
+
+   /* VS output information */
+   unsigned num_clip_distances;
+   unsigned num_cull_distances;
+   vs_output_state vs_output;
+
+   /* Streamout */
+   Temp streamout_buffers = Temp(0, s1);
+   Temp streamout_write_idx = Temp(0, s1);
+   Temp streamout_config = Temp(0, s1);
+   Temp streamout_offset[4] = {Temp(0, s1), Temp(0, s1), Temp(0, s1), Temp(0, s1)};
+};
+
+fs_input get_interp_input(nir_intrinsic_op intrin, enum glsl_interp_mode interp)
+{
+   switch (interp) {
+   case INTERP_MODE_SMOOTH:
+   case INTERP_MODE_NONE:
+      if (intrin == nir_intrinsic_load_barycentric_pixel ||
+          intrin == nir_intrinsic_load_barycentric_at_sample ||
+          intrin == nir_intrinsic_load_barycentric_at_offset)
+         return fs_input::persp_center_p1;
+      else if (intrin == nir_intrinsic_load_barycentric_centroid)
+         return fs_input::persp_centroid_p1;
+      else if (intrin == nir_intrinsic_load_barycentric_sample)
+         return fs_input::persp_sample_p1;
+      break;
+   case INTERP_MODE_NOPERSPECTIVE:
+      if (intrin == nir_intrinsic_load_barycentric_pixel)
+         return fs_input::linear_center_p1;
+      else if (intrin == nir_intrinsic_load_barycentric_centroid)
+         return fs_input::linear_centroid_p1;
+      else if (intrin == nir_intrinsic_load_barycentric_sample)
+         return fs_input::linear_sample_p1;
+      break;
+   default:
+      break;
+   }
+   return fs_input::max_inputs;
+}
+
+void init_context(isel_context *ctx, nir_shader *shader)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+
+   ctx->shader = shader;
+   ctx->divergent_vals = nir_divergence_analysis(shader, nir_divergence_view_index_uniform);
+
+   std::unique_ptr<Temp[]> allocated{new Temp[impl->ssa_alloc]()};
+   memset(&ctx->fs_vgpr_args, false, sizeof(ctx->fs_vgpr_args));
+
+   bool done = false;
+   while (!done) {
+      done = true;
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            switch(instr->type) {
+            case nir_instr_type_alu: {
+               nir_alu_instr *alu_instr = nir_instr_as_alu(instr);
+               unsigned size =  alu_instr->dest.dest.ssa.num_components;
+               if (alu_instr->dest.dest.ssa.bit_size == 64)
+                  size *= 2;
+               RegType type = RegType::sgpr;
+               switch(alu_instr->op) {
+                  case nir_op_fmul:
+                  case nir_op_fadd:
+                  case nir_op_fsub:
+                  case nir_op_fmax:
+                  case nir_op_fmin:
+                  case nir_op_fmax3:
+                  case nir_op_fmin3:
+                  case nir_op_fmed3:
+                  case nir_op_fmod:
+                  case nir_op_frem:
+                  case nir_op_fneg:
+                  case nir_op_fabs:
+                  case nir_op_fsat:
+                  case nir_op_fsign:
+                  case nir_op_frcp:
+                  case nir_op_frsq:
+                  case nir_op_fsqrt:
+                  case nir_op_fexp2:
+                  case nir_op_flog2:
+                  case nir_op_ffract:
+                  case nir_op_ffloor:
+                  case nir_op_fceil:
+                  case nir_op_ftrunc:
+                  case nir_op_fround_even:
+                  case nir_op_fsin:
+                  case nir_op_fcos:
+                  case nir_op_f2f32:
+                  case nir_op_f2f64:
+                  case nir_op_u2f32:
+                  case nir_op_u2f64:
+                  case nir_op_i2f32:
+                  case nir_op_i2f64:
+                  case nir_op_pack_half_2x16:
+                  case nir_op_unpack_half_2x16_split_x:
+                  case nir_op_unpack_half_2x16_split_y:
+                  case nir_op_fddx:
+                  case nir_op_fddy:
+                  case nir_op_fddx_fine:
+                  case nir_op_fddy_fine:
+                  case nir_op_fddx_coarse:
+                  case nir_op_fddy_coarse:
+                  case nir_op_fquantize2f16:
+                  case nir_op_ldexp:
+                  case nir_op_frexp_sig:
+                  case nir_op_frexp_exp:
+                  case nir_op_cube_face_index:
+                  case nir_op_cube_face_coord:
+                     type = RegType::vgpr;
+                     break;
+                  case nir_op_flt:
+                  case nir_op_fge:
+                  case nir_op_feq:
+                  case nir_op_fne:
+                     size = 2;
+                     break;
+                  case nir_op_ilt:
+                  case nir_op_ige:
+                  case nir_op_ult:
+                  case nir_op_uge:
+                     size = alu_instr->src[0].src.ssa->bit_size == 64 ? 2 : 1;
+                     /* fallthrough */
+                  case nir_op_ieq:
+                  case nir_op_ine:
+                  case nir_op_i2b1:
+                     if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index]) {
+                        size = 2;
+                     } else {
+                        for (unsigned i = 0; i < nir_op_infos[alu_instr->op].num_inputs; i++) {
+                           if (allocated[alu_instr->src[i].src.ssa->index].type() == RegType::vgpr)
+                              size = 2;
+                        }
+                     }
+                     break;
+                  case nir_op_f2i64:
+                  case nir_op_f2u64:
+                  case nir_op_b2i32:
+                  case nir_op_b2f32:
+                  case nir_op_f2i32:
+                  case nir_op_f2u32:
+                     type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
+                     break;
+                  case nir_op_bcsel:
+                     if (alu_instr->dest.dest.ssa.bit_size == 1) {
+                        if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index])
+                           size = 2;
+                        else if (allocated[alu_instr->src[1].src.ssa->index].regClass() == s2 &&
+                                 allocated[alu_instr->src[2].src.ssa->index].regClass() == s2)
+                           size = 2;
+                        else
+                           size = 1;
+                     } else {
+                        if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index]) {
+                           type = RegType::vgpr;
+                        } else {
+                           if (allocated[alu_instr->src[1].src.ssa->index].type() == RegType::vgpr ||
+                               allocated[alu_instr->src[2].src.ssa->index].type() == RegType::vgpr) {
+                              type = RegType::vgpr;
+                           }
+                        }
+                        if (alu_instr->src[1].src.ssa->num_components == 1 && alu_instr->src[2].src.ssa->num_components == 1) {
+                           assert(allocated[alu_instr->src[1].src.ssa->index].size() == allocated[alu_instr->src[2].src.ssa->index].size());
+                           size = allocated[alu_instr->src[1].src.ssa->index].size();
+                        }
+                     }
+                     break;
+                  case nir_op_mov:
+                     if (alu_instr->dest.dest.ssa.bit_size == 1) {
+                        size = allocated[alu_instr->src[0].src.ssa->index].size();
+                     } else {
+                        type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
+                     }
+                     break;
+                  case nir_op_inot:
+                  case nir_op_ixor:
+                     if (alu_instr->dest.dest.ssa.bit_size == 1) {
+                        size = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? 2 : 1;
+                        break;
+                     } else {
+                        /* fallthrough */
+                     }
+                  default:
+                     if (alu_instr->dest.dest.ssa.bit_size == 1) {
+                        if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index]) {
+                           size = 2;
+                        } else {
+                           size = 2;
+                           for (unsigned i = 0; i < nir_op_infos[alu_instr->op].num_inputs; i++) {
+                              if (allocated[alu_instr->src[i].src.ssa->index].regClass() == s1) {
+                                 size = 1;
+                                 break;
+                              }
+                           }
+                        }
+                     } else {
+                        for (unsigned i = 0; i < nir_op_infos[alu_instr->op].num_inputs; i++) {
+                           if (allocated[alu_instr->src[i].src.ssa->index].type() == RegType::vgpr)
+                              type = RegType::vgpr;
+                        }
+                     }
+                     break;
+               }
+               allocated[alu_instr->dest.dest.ssa.index] = Temp(0, RegClass(type, size));
+               break;
+            }
+            case nir_instr_type_load_const: {
+               unsigned size = nir_instr_as_load_const(instr)->def.num_components;
+               if (nir_instr_as_load_const(instr)->def.bit_size == 64)
+                  size *= 2;
+               allocated[nir_instr_as_load_const(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
+               break;
+            }
+            case nir_instr_type_intrinsic: {
+               nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
+               if (!nir_intrinsic_infos[intrinsic->intrinsic].has_dest)
+                  break;
+               unsigned size =  intrinsic->dest.ssa.num_components;
+               if (intrinsic->dest.ssa.bit_size == 64)
+                  size *= 2;
+               RegType type = RegType::sgpr;
+               switch(intrinsic->intrinsic) {
+                  case nir_intrinsic_load_push_constant:
+                  case nir_intrinsic_load_work_group_id:
+                  case nir_intrinsic_load_num_work_groups:
+                  case nir_intrinsic_load_subgroup_id:
+                  case nir_intrinsic_load_num_subgroups:
+                  case nir_intrinsic_load_first_vertex:
+                  case nir_intrinsic_load_base_instance:
+                  case nir_intrinsic_get_buffer_size:
+                  case nir_intrinsic_vote_all:
+                  case nir_intrinsic_vote_any:
+                  case nir_intrinsic_read_first_invocation:
+                  case nir_intrinsic_read_invocation:
+                  case nir_intrinsic_first_invocation:
+                  case nir_intrinsic_vulkan_resource_index:
+                     type = RegType::sgpr;
+                     break;
+                  case nir_intrinsic_ballot:
+                     type = RegType::sgpr;
+                     size = 2;
+                     break;
+                  case nir_intrinsic_load_sample_id:
+                  case nir_intrinsic_load_sample_mask_in:
+                  case nir_intrinsic_load_input:
+                  case nir_intrinsic_load_vertex_id:
+                  case nir_intrinsic_load_vertex_id_zero_base:
+                  case nir_intrinsic_load_barycentric_sample:
+                  case nir_intrinsic_load_barycentric_pixel:
+                  case nir_intrinsic_load_barycentric_centroid:
+                  case nir_intrinsic_load_barycentric_at_sample:
+                  case nir_intrinsic_load_barycentric_at_offset:
+                  case nir_intrinsic_load_interpolated_input:
+                  case nir_intrinsic_load_frag_coord:
+                  case nir_intrinsic_load_sample_pos:
+                  case nir_intrinsic_load_layer_id:
+                  case nir_intrinsic_load_local_invocation_id:
+                  case nir_intrinsic_load_local_invocation_index:
+                  case nir_intrinsic_load_subgroup_invocation:
+                  case nir_intrinsic_write_invocation_amd:
+                  case nir_intrinsic_mbcnt_amd:
+                  case nir_intrinsic_load_instance_id:
+                  case nir_intrinsic_ssbo_atomic_add:
+                  case nir_intrinsic_ssbo_atomic_imin:
+                  case nir_intrinsic_ssbo_atomic_umin:
+                  case nir_intrinsic_ssbo_atomic_imax:
+                  case nir_intrinsic_ssbo_atomic_umax:
+                  case nir_intrinsic_ssbo_atomic_and:
+                  case nir_intrinsic_ssbo_atomic_or:
+                  case nir_intrinsic_ssbo_atomic_xor:
+                  case nir_intrinsic_ssbo_atomic_exchange:
+                  case nir_intrinsic_ssbo_atomic_comp_swap:
+                  case nir_intrinsic_image_deref_atomic_add:
+                  case nir_intrinsic_image_deref_atomic_umin:
+                  case nir_intrinsic_image_deref_atomic_imin:
+                  case nir_intrinsic_image_deref_atomic_umax:
+                  case nir_intrinsic_image_deref_atomic_imax:
+                  case nir_intrinsic_image_deref_atomic_and:
+                  case nir_intrinsic_image_deref_atomic_or:
+                  case nir_intrinsic_image_deref_atomic_xor:
+                  case nir_intrinsic_image_deref_atomic_exchange:
+                  case nir_intrinsic_image_deref_atomic_comp_swap:
+                  case nir_intrinsic_image_deref_size:
+                  case nir_intrinsic_shared_atomic_add:
+                  case nir_intrinsic_shared_atomic_imin:
+                  case nir_intrinsic_shared_atomic_umin:
+                  case nir_intrinsic_shared_atomic_imax:
+                  case nir_intrinsic_shared_atomic_umax:
+                  case nir_intrinsic_shared_atomic_and:
+                  case nir_intrinsic_shared_atomic_or:
+                  case nir_intrinsic_shared_atomic_xor:
+                  case nir_intrinsic_shared_atomic_exchange:
+                  case nir_intrinsic_shared_atomic_comp_swap:
+                  case nir_intrinsic_load_scratch:
+                     type = RegType::vgpr;
+                     break;
+                  case nir_intrinsic_shuffle:
+                  case nir_intrinsic_quad_broadcast:
+                  case nir_intrinsic_quad_swap_horizontal:
+                  case nir_intrinsic_quad_swap_vertical:
+                  case nir_intrinsic_quad_swap_diagonal:
+                  case nir_intrinsic_quad_swizzle_amd:
+                  case nir_intrinsic_masked_swizzle_amd:
+                  case nir_intrinsic_inclusive_scan:
+                  case nir_intrinsic_exclusive_scan:
+                     if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
+                        type = RegType::sgpr;
+                     } else if (intrinsic->src[0].ssa->bit_size == 1) {
+                        type = RegType::sgpr;
+                        size = 2;
+                     } else {
+                        type = RegType::vgpr;
+                     }
+                     break;
+                  case nir_intrinsic_load_view_index:
+                     type = ctx->stage == fragment_fs ? RegType::vgpr : RegType::sgpr;
+                     break;
+                  case nir_intrinsic_load_front_face:
+                  case nir_intrinsic_load_helper_invocation:
+                  case nir_intrinsic_is_helper_invocation:
+                     type = RegType::sgpr;
+                     size = 2;
+                     break;
+                  case nir_intrinsic_reduce:
+                     if (nir_intrinsic_cluster_size(intrinsic) == 0 ||
+                         !ctx->divergent_vals[intrinsic->dest.ssa.index]) {
+                        type = RegType::sgpr;
+                     } else if (intrinsic->src[0].ssa->bit_size == 1) {
+                        type = RegType::sgpr;
+                        size = 2;
+                     } else {
+                        type = RegType::vgpr;
+                     }
+                     break;
+                  case nir_intrinsic_load_ubo:
+                  case nir_intrinsic_load_ssbo:
+                  case nir_intrinsic_load_global:
+                     type = ctx->divergent_vals[intrinsic->dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
+                     break;
+                  /* due to copy propagation, the swizzled imov is removed if num dest components == 1 */
+                  case nir_intrinsic_load_shared:
+                     if (ctx->divergent_vals[intrinsic->dest.ssa.index])
+                        type = RegType::vgpr;
+                     else
+                        type = RegType::sgpr;
+                     break;
+                  default:
+                     for (unsigned i = 0; i < nir_intrinsic_infos[intrinsic->intrinsic].num_srcs; i++) {
+                        if (allocated[intrinsic->src[i].ssa->index].type() == RegType::vgpr)
+                           type = RegType::vgpr;
+                     }
+                     break;
+               }
+               allocated[intrinsic->dest.ssa.index] = Temp(0, RegClass(type, size));
+
+               switch(intrinsic->intrinsic) {
+                  case nir_intrinsic_load_barycentric_sample:
+                  case nir_intrinsic_load_barycentric_pixel:
+                  case nir_intrinsic_load_barycentric_centroid:
+                  case nir_intrinsic_load_barycentric_at_sample:
+                  case nir_intrinsic_load_barycentric_at_offset: {
+                     glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(intrinsic);
+                     ctx->fs_vgpr_args[get_interp_input(intrinsic->intrinsic, mode)] = true;
+                     break;
+                  }
+                  case nir_intrinsic_load_front_face:
+                     ctx->fs_vgpr_args[fs_input::front_face] = true;
+                     break;
+                  case nir_intrinsic_load_frag_coord:
+                  case nir_intrinsic_load_sample_pos: {
+                     uint8_t mask = nir_ssa_def_components_read(&intrinsic->dest.ssa);
+                     for (unsigned i = 0; i < 4; i++) {
+                        if (mask & (1 << i))
+                           ctx->fs_vgpr_args[fs_input::frag_pos_0 + i] = true;
+
+                     }
+                     break;
+                  }
+                  case nir_intrinsic_load_sample_id:
+                     ctx->fs_vgpr_args[fs_input::ancillary] = true;
+                     break;
+                  case nir_intrinsic_load_sample_mask_in:
+                     ctx->fs_vgpr_args[fs_input::ancillary] = true;
+                     ctx->fs_vgpr_args[fs_input::sample_coverage] = true;
+                     break;
+                  default:
+                     break;
+               }
+               break;
+            }
+            case nir_instr_type_tex: {
+               nir_tex_instr* tex = nir_instr_as_tex(instr);
+               unsigned size = tex->dest.ssa.num_components;
+
+               if (tex->dest.ssa.bit_size == 64)
+                  size *= 2;
+               if (tex->op == nir_texop_texture_samples)
+                  assert(!ctx->divergent_vals[tex->dest.ssa.index]);
+               if (ctx->divergent_vals[tex->dest.ssa.index])
+                  allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::vgpr, size));
+               else
+                  allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::sgpr, size));
+               break;
+            }
+            case nir_instr_type_parallel_copy: {
+               nir_foreach_parallel_copy_entry(entry, nir_instr_as_parallel_copy(instr)) {
+                  allocated[entry->dest.ssa.index] = allocated[entry->src.ssa->index];
+               }
+               break;
+            }
+            case nir_instr_type_ssa_undef: {
+               unsigned size = nir_instr_as_ssa_undef(instr)->def.num_components;
+               if (nir_instr_as_ssa_undef(instr)->def.bit_size == 64)
+                  size *= 2;
+               allocated[nir_instr_as_ssa_undef(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
+               break;
+            }
+            case nir_instr_type_phi: {
+               nir_phi_instr* phi = nir_instr_as_phi(instr);
+               RegType type;
+               unsigned size = phi->dest.ssa.num_components;
+
+               if (phi->dest.ssa.bit_size == 1) {
+                  assert(size == 1 && "multiple components not yet supported on boolean phis.");
+                  type = RegType::sgpr;
+                  size *= ctx->divergent_vals[phi->dest.ssa.index] ? 2 : 1;
+                  allocated[phi->dest.ssa.index] = Temp(0, RegClass(type, size));
+                  break;
+               }
+
+               if (ctx->divergent_vals[phi->dest.ssa.index]) {
+                  type = RegType::vgpr;
+               } else {
+                  type = RegType::sgpr;
+                  nir_foreach_phi_src (src, phi) {
+                     if (allocated[src->src.ssa->index].type() == RegType::vgpr)
+                        type = RegType::vgpr;
+                     if (allocated[src->src.ssa->index].type() == RegType::none)
+                        done = false;
+                  }
+               }
+
+               size *= phi->dest.ssa.bit_size == 64 ? 2 : 1;
+               RegClass rc = RegClass(type, size);
+               if (rc != allocated[phi->dest.ssa.index].regClass()) {
+                  done = false;
+               } else {
+                  nir_foreach_phi_src(src, phi)
+                     assert(allocated[src->src.ssa->index].size() == rc.size());
+               }
+               allocated[phi->dest.ssa.index] = Temp(0, rc);
+               break;
+            }
+            default:
+               break;
+            }
+         }
+      }
+   }
+
+   for (unsigned i = 0; i < impl->ssa_alloc; i++)
+      allocated[i] = Temp(ctx->program->allocateId(), allocated[i].regClass());
+
+   ctx->allocated.reset(allocated.release());
+}
+
+struct user_sgpr_info {
+   uint8_t num_sgpr;
+   uint8_t remaining_sgprs;
+   uint8_t user_sgpr_idx;
+   bool need_ring_offsets;
+   bool indirect_all_descriptor_sets;
+};
+
+static void allocate_inline_push_consts(isel_context *ctx,
+                                        user_sgpr_info& user_sgpr_info)
+{
+   uint8_t remaining_sgprs = user_sgpr_info.remaining_sgprs;
+
+   /* Only supported if shaders use push constants. */
+   if (ctx->program->info->min_push_constant_used == UINT8_MAX)
+      return;
+
+   /* Only supported if shaders don't have indirect push constants. */
+   if (ctx->program->info->has_indirect_push_constants)
+      return;
+
+   /* Only supported for 32-bit push constants. */
+   //TODO: it's possible that some day, the load/store vectorization could make this inaccurate
+   if (!ctx->program->info->has_only_32bit_push_constants)
+      return;
+
+   uint8_t num_push_consts =
+      (ctx->program->info->max_push_constant_used -
+       ctx->program->info->min_push_constant_used) / 4;
+
+   /* Check if the number of user SGPRs is large enough. */
+   if (num_push_consts < remaining_sgprs) {
+      ctx->program->info->num_inline_push_consts = num_push_consts;
+   } else {
+      ctx->program->info->num_inline_push_consts = remaining_sgprs;
+   }
+
+   /* Clamp to the maximum number of allowed inlined push constants. */
+   if (ctx->program->info->num_inline_push_consts > MAX_INLINE_PUSH_CONSTS)
+      ctx->program->info->num_inline_push_consts = MAX_INLINE_PUSH_CONSTS;
+
+   if (ctx->program->info->num_inline_push_consts == num_push_consts &&
+       !ctx->program->info->loads_dynamic_offsets) {
+      /* Disable the default push constants path if all constants are
+       * inlined and if shaders don't use dynamic descriptors.
+       */
+      ctx->program->info->loads_push_constants = false;
+      user_sgpr_info.num_sgpr--;
+      user_sgpr_info.remaining_sgprs++;
+   }
+
+   ctx->program->info->base_inline_push_consts =
+      ctx->program->info->min_push_constant_used / 4;
+
+   user_sgpr_info.num_sgpr += ctx->program->info->num_inline_push_consts;
+   user_sgpr_info.remaining_sgprs -= ctx->program->info->num_inline_push_consts;
+}
+
+static void allocate_user_sgprs(isel_context *ctx,
+                                bool needs_view_index, user_sgpr_info& user_sgpr_info)
+{
+   memset(&user_sgpr_info, 0, sizeof(struct user_sgpr_info));
+   uint32_t user_sgpr_count = 0;
+
+   /* until we sort out scratch/global buffers always assign ring offsets for gs/vs/es */
+   if (ctx->stage != fragment_fs &&
+       ctx->stage != compute_cs
+       /*|| ctx->is_gs_copy_shader */)
+      user_sgpr_info.need_ring_offsets = true;
+
+   if (ctx->stage == fragment_fs &&
+       ctx->program->info->ps.needs_sample_positions)
+      user_sgpr_info.need_ring_offsets = true;
+
+   /* 2 user sgprs will nearly always be allocated for scratch/rings */
+   if (ctx->options->supports_spill || user_sgpr_info.need_ring_offsets || ctx->scratch_enabled)
+      user_sgpr_count += 2;
+
+   switch (ctx->stage) {
+   case vertex_vs:
+   /* if (!ctx->is_gs_copy_shader) */ {
+         if (ctx->program->info->vs.has_vertex_buffers)
+            user_sgpr_count++;
+         user_sgpr_count += ctx->program->info->vs.needs_draw_id ? 3 : 2;
+      }
+      break;
+   case fragment_fs:
+      //user_sgpr_count += ctx->program->info->ps.needs_sample_positions;
+      break;
+   case compute_cs:
+      if (ctx->program->info->cs.uses_grid_size)
+         user_sgpr_count += 3;
+      break;
+   default:
+      unreachable("Shader stage not implemented");
+   }
+
+   if (needs_view_index)
+      user_sgpr_count++;
+
+   if (ctx->program->info->loads_push_constants)
+      user_sgpr_count += 1; /* we use 32bit pointers */
+
+   if (ctx->program->info->so.num_outputs)
+      user_sgpr_count += 1; /* we use 32bit pointers */
+
+   uint32_t available_sgprs = ctx->options->chip_class >= GFX9 && !(ctx->stage & hw_cs) ? 32 : 16;
+   uint32_t remaining_sgprs = available_sgprs - user_sgpr_count;
+   uint32_t num_desc_set = util_bitcount(ctx->program->info->desc_set_used_mask);
+
+   if (available_sgprs < user_sgpr_count + num_desc_set) {
+      user_sgpr_info.indirect_all_descriptor_sets = true;
+      user_sgpr_info.num_sgpr = user_sgpr_count + 1;
+      user_sgpr_info.remaining_sgprs = remaining_sgprs - 1;
+   } else {
+      user_sgpr_info.num_sgpr = user_sgpr_count + num_desc_set;
+      user_sgpr_info.remaining_sgprs = remaining_sgprs - num_desc_set;
+   }
+
+   allocate_inline_push_consts(ctx, user_sgpr_info);
+}
+
+#define MAX_ARGS 64
+struct arg_info {
+   RegClass types[MAX_ARGS];
+   Temp *assign[MAX_ARGS];
+   PhysReg reg[MAX_ARGS];
+   unsigned array_params_mask;
+   uint8_t count;
+   uint8_t sgpr_count;
+   uint8_t num_sgprs_used;
+   uint8_t num_vgprs_used;
+};
+
+static void
+add_arg(arg_info *info, RegClass rc, Temp *param_ptr, unsigned reg)
+{
+   assert(info->count < MAX_ARGS);
+
+   info->assign[info->count] = param_ptr;
+   info->types[info->count] = rc;
+
+   if (rc.type() == RegType::sgpr) {
+      info->num_sgprs_used += rc.size();
+      info->sgpr_count++;
+      info->reg[info->count] = PhysReg{reg};
+   } else {
+      assert(rc.type() == RegType::vgpr);
+      info->num_vgprs_used += rc.size();
+      info->reg[info->count] = PhysReg{reg + 256};
+   }
+   info->count++;
+}
+
+static void
+set_loc(struct radv_userdata_info *ud_info, uint8_t *sgpr_idx, uint8_t num_sgprs)
+{
+   ud_info->sgpr_idx = *sgpr_idx;
+   ud_info->num_sgprs = num_sgprs;
+   *sgpr_idx += num_sgprs;
+}
+
+static void
+set_loc_shader(isel_context *ctx, int idx, uint8_t *sgpr_idx,
+               uint8_t num_sgprs)
+{
+   struct radv_userdata_info *ud_info = &ctx->program->info->user_sgprs_locs.shader_data[idx];
+   assert(ud_info);
+
+   set_loc(ud_info, sgpr_idx, num_sgprs);
+}
+
+static void
+set_loc_shader_ptr(isel_context *ctx, int idx, uint8_t *sgpr_idx)
+{
+   bool use_32bit_pointers = idx != AC_UD_SCRATCH_RING_OFFSETS;
+
+   set_loc_shader(ctx, idx, sgpr_idx, use_32bit_pointers ? 1 : 2);
+}
+
+static void
+set_loc_desc(isel_context *ctx, int idx,  uint8_t *sgpr_idx)
+{
+   struct radv_userdata_locations *locs = &ctx->program->info->user_sgprs_locs;
+   struct radv_userdata_info *ud_info = &locs->descriptor_sets[idx];
+   assert(ud_info);
+
+   set_loc(ud_info, sgpr_idx, 1);
+   locs->descriptor_sets_enabled |= 1 << idx;
+}
+
+static void
+declare_global_input_sgprs(isel_context *ctx,
+                           /* bool has_previous_stage, gl_shader_stage previous_stage, */
+                           user_sgpr_info *user_sgpr_info,
+                           struct arg_info *args,
+                           Temp *desc_sets)
+{
+   /* 1 for each descriptor set */
+   if (!user_sgpr_info->indirect_all_descriptor_sets) {
+      uint32_t mask = ctx->program->info->desc_set_used_mask;
+      while (mask) {
+         int i = u_bit_scan(&mask);
+         add_arg(args, s1, &desc_sets[i], user_sgpr_info->user_sgpr_idx);
+         set_loc_desc(ctx, i, &user_sgpr_info->user_sgpr_idx);
+      }
+      /* NIR->LLVM might have set this to true if RADV_DEBUG=compiletime */
+      ctx->program->info->need_indirect_descriptor_sets = false;
+   } else {
+      add_arg(args, s1, desc_sets, user_sgpr_info->user_sgpr_idx);
+      set_loc_shader_ptr(ctx, AC_UD_INDIRECT_DESCRIPTOR_SETS, &user_sgpr_info->user_sgpr_idx);
+      ctx->program->info->need_indirect_descriptor_sets = true;
+   }
+
+   if (ctx->program->info->loads_push_constants) {
+      /* 1 for push constants and dynamic descriptors */
+      add_arg(args, s1, &ctx->push_constants, user_sgpr_info->user_sgpr_idx);
+      set_loc_shader_ptr(ctx, AC_UD_PUSH_CONSTANTS, &user_sgpr_info->user_sgpr_idx);
+   }
+
+   if (ctx->program->info->num_inline_push_consts) {
+      unsigned count = ctx->program->info->num_inline_push_consts;
+      for (unsigned i = 0; i < count; i++)
+         add_arg(args, s1, &ctx->inline_push_consts[i], user_sgpr_info->user_sgpr_idx + i);
+      set_loc_shader(ctx, AC_UD_INLINE_PUSH_CONSTANTS, &user_sgpr_info->user_sgpr_idx, count);
+
+      ctx->num_inline_push_consts = ctx->program->info->num_inline_push_consts;
+      ctx->base_inline_push_consts = ctx->program->info->base_inline_push_consts;
+   }
+
+   if (ctx->program->info->so.num_outputs) {
+      add_arg(args, s1, &ctx->streamout_buffers, user_sgpr_info->user_sgpr_idx);
+      set_loc_shader_ptr(ctx, AC_UD_STREAMOUT_BUFFERS, &user_sgpr_info->user_sgpr_idx);
+   }
+}
+
+static void
+declare_vs_input_vgprs(isel_context *ctx, struct arg_info *args)
+{
+   unsigned vgpr_idx = 0;
+   add_arg(args, v1, &ctx->vertex_id, vgpr_idx++);
+/* if (!ctx->is_gs_copy_shader) */ {
+      if (ctx->options->key.vs.out.as_ls) {
+         add_arg(args, v1, &ctx->rel_auto_id, vgpr_idx++);
+         add_arg(args, v1, &ctx->instance_id, vgpr_idx++);
+      } else {
+         add_arg(args, v1, &ctx->instance_id, vgpr_idx++);
+         add_arg(args, v1, &ctx->vs_prim_id, vgpr_idx++);
+      }
+      add_arg(args, v1, NULL, vgpr_idx); /* unused */
+   }
+}
+
+static void
+declare_streamout_sgprs(isel_context *ctx, struct arg_info *args, unsigned *idx)
+{
+   /* Streamout SGPRs. */
+   if (ctx->program->info->so.num_outputs) {
+      assert(ctx->stage & hw_vs);
+
+      if (ctx->stage != tess_eval_vs) {
+         add_arg(args, s1, &ctx->streamout_config, (*idx)++);
+      } else {
+         args->assign[args->count - 1] = &ctx->streamout_config;
+         args->types[args->count - 1] = s1;
+      }
+
+      add_arg(args, s1, &ctx->streamout_write_idx, (*idx)++);
+   }
+
+   /* A streamout buffer offset is loaded if the stride is non-zero. */
+   for (unsigned i = 0; i < 4; i++) {
+      if (!ctx->program->info->so.strides[i])
+         continue;
+
+      add_arg(args, s1, &ctx->streamout_offset[i], (*idx)++);
+   }
+}
+
+static bool needs_view_index_sgpr(isel_context *ctx)
+{
+   switch (ctx->stage) {
+   case vertex_vs:
+      return ctx->program->info->needs_multiview_view_index || ctx->options->key.has_multiview_view_index;
+   case tess_eval_vs:
+      return ctx->program->info->needs_multiview_view_index && ctx->options->key.has_multiview_view_index;
+   case vertex_ls:
+   case vertex_tess_control_ls:
+   case vertex_geometry_es:
+   case tess_control_hs:
+   case tess_eval_es:
+   case tess_eval_geometry_es:
+   case geometry_gs:
+      return ctx->program->info->needs_multiview_view_index;
+   default:
+      return false;
+   }
+}
+
+static inline bool
+add_fs_arg(isel_context *ctx, arg_info *args, unsigned &vgpr_idx, fs_input input, unsigned value, bool enable_next = false, RegClass rc = v1)
+{
+   if (!ctx->fs_vgpr_args[input])
+      return false;
+
+   add_arg(args, rc, &ctx->fs_inputs[input], vgpr_idx);
+   vgpr_idx += rc.size();
+
+   if (enable_next) {
+      add_arg(args, rc, &ctx->fs_inputs[input + 1], vgpr_idx);
+      vgpr_idx += rc.size();
+   }
+
+   ctx->program->config->spi_ps_input_addr |= value;
+   ctx->program->config->spi_ps_input_ena |= value;
+   return true;
+}
+
+void add_startpgm(struct isel_context *ctx)
+{
+   user_sgpr_info user_sgpr_info;
+   bool needs_view_index = needs_view_index_sgpr(ctx);
+   allocate_user_sgprs(ctx, needs_view_index, user_sgpr_info);
+   arg_info args = {};
+
+   /* this needs to be in sgprs 0 and 1 */
+   if (ctx->options->supports_spill || user_sgpr_info.need_ring_offsets || ctx->scratch_enabled) {
+      add_arg(&args, s2, &ctx->private_segment_buffer, 0);
+      set_loc_shader_ptr(ctx, AC_UD_SCRATCH_RING_OFFSETS, &user_sgpr_info.user_sgpr_idx);
+   }
+
+   unsigned vgpr_idx = 0;
+   switch (ctx->stage) {
+   case vertex_vs: {
+      declare_global_input_sgprs(ctx, &user_sgpr_info, &args, ctx->descriptor_sets);
+      if (ctx->program->info->vs.has_vertex_buffers) {
+         add_arg(&args, s1, &ctx->vertex_buffers, user_sgpr_info.user_sgpr_idx);
+         set_loc_shader_ptr(ctx, AC_UD_VS_VERTEX_BUFFERS, &user_sgpr_info.user_sgpr_idx);
+      }
+      add_arg(&args, s1, &ctx->base_vertex, user_sgpr_info.user_sgpr_idx);
+      add_arg(&args, s1, &ctx->start_instance, user_sgpr_info.user_sgpr_idx + 1);
+      if (ctx->program->info->vs.needs_draw_id) {
+         add_arg(&args, s1, &ctx->draw_id, user_sgpr_info.user_sgpr_idx + 2);
+         set_loc_shader(ctx, AC_UD_VS_BASE_VERTEX_START_INSTANCE, &user_sgpr_info.user_sgpr_idx, 3);
+      } else
+         set_loc_shader(ctx, AC_UD_VS_BASE_VERTEX_START_INSTANCE, &user_sgpr_info.user_sgpr_idx, 2);
+
+      if (needs_view_index) {
+         add_arg(&args, s1, &ctx->view_index, user_sgpr_info.user_sgpr_idx);
+         set_loc_shader(ctx, AC_UD_VIEW_INDEX, &user_sgpr_info.user_sgpr_idx, 1);
+      }
+
+      assert(user_sgpr_info.user_sgpr_idx == user_sgpr_info.num_sgpr);
+      unsigned idx = user_sgpr_info.user_sgpr_idx;
+      if (ctx->options->key.vs.out.as_es)
+         add_arg(&args, s1, &ctx->es2gs_offset, idx++);
+      else
+         declare_streamout_sgprs(ctx, &args, &idx);
+
+      if (ctx->scratch_enabled)
+         add_arg(&args, s1, &ctx->scratch_offset, idx++);
+
+      declare_vs_input_vgprs(ctx, &args);
+      break;
+   }
+   case fragment_fs: {
+      declare_global_input_sgprs(ctx, &user_sgpr_info, &args, ctx->descriptor_sets);
+
+      assert(user_sgpr_info.user_sgpr_idx == user_sgpr_info.num_sgpr);
+      add_arg(&args, s1, &ctx->prim_mask, user_sgpr_info.user_sgpr_idx);
+
+      if (ctx->scratch_enabled)
+         add_arg(&args, s1, &ctx->scratch_offset, user_sgpr_info.user_sgpr_idx + 1);
+
+      ctx->program->config->spi_ps_input_addr = 0;
+      ctx->program->config->spi_ps_input_ena = 0;
+
+      bool has_interp_mode = false;
+
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::persp_sample_p1, S_0286CC_PERSP_SAMPLE_ENA(1), true);
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::persp_center_p1, S_0286CC_PERSP_CENTER_ENA(1), true);
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::persp_centroid_p1, S_0286CC_PERSP_CENTROID_ENA(1), true);
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::persp_pull_model, S_0286CC_PERSP_PULL_MODEL_ENA(1), false, v3);
+
+      if (!has_interp_mode && ctx->fs_vgpr_args[fs_input::frag_pos_3]) {
+         /* If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be enabled too */
+         ctx->fs_vgpr_args[fs_input::persp_center_p1] = true;
+         has_interp_mode = add_fs_arg(ctx, &args, vgpr_idx, fs_input::persp_center_p1, S_0286CC_PERSP_CENTER_ENA(1), true);
+      }
+
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::linear_sample_p1, S_0286CC_LINEAR_SAMPLE_ENA(1), true);
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::linear_center_p1, S_0286CC_LINEAR_CENTER_ENA(1), true);
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::linear_centroid_p1, S_0286CC_LINEAR_CENTROID_ENA(1), true);
+      has_interp_mode |= add_fs_arg(ctx, &args, vgpr_idx, fs_input::line_stipple, S_0286CC_LINE_STIPPLE_TEX_ENA(1));
+
+      if (!has_interp_mode) {
+         /* At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled */
+         ctx->fs_vgpr_args[fs_input::persp_center_p1] = true;
+         has_interp_mode = add_fs_arg(ctx, &args, vgpr_idx, fs_input::persp_center_p1, S_0286CC_PERSP_CENTER_ENA(1), true);
+      }
+
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::frag_pos_0, S_0286CC_POS_X_FLOAT_ENA(1));
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::frag_pos_1, S_0286CC_POS_Y_FLOAT_ENA(1));
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::frag_pos_2, S_0286CC_POS_Z_FLOAT_ENA(1));
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::frag_pos_3, S_0286CC_POS_W_FLOAT_ENA(1));
+
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::front_face, S_0286CC_FRONT_FACE_ENA(1));
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::ancillary, S_0286CC_ANCILLARY_ENA(1));
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::sample_coverage, S_0286CC_SAMPLE_COVERAGE_ENA(1));
+      add_fs_arg(ctx, &args, vgpr_idx, fs_input::fixed_pt, S_0286CC_POS_FIXED_PT_ENA(1));
+
+      ASSERTED bool unset_interp_mode = !(ctx->program->config->spi_ps_input_addr & 0x7F) ||
+                                        (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_addr)
+                                        && !(ctx->program->config->spi_ps_input_addr & 0xF));
+
+      assert(has_interp_mode);
+      assert(!unset_interp_mode);
+      break;
+   }
+   case compute_cs: {
+      declare_global_input_sgprs(ctx, &user_sgpr_info, &args, ctx->descriptor_sets);
+
+      if (ctx->program->info->cs.uses_grid_size) {
+         add_arg(&args, s1, &ctx->num_workgroups[0], user_sgpr_info.user_sgpr_idx);
+         add_arg(&args, s1, &ctx->num_workgroups[1], user_sgpr_info.user_sgpr_idx + 1);
+         add_arg(&args, s1, &ctx->num_workgroups[2], user_sgpr_info.user_sgpr_idx + 2);
+         set_loc_shader(ctx, AC_UD_CS_GRID_SIZE, &user_sgpr_info.user_sgpr_idx, 3);
+      }
+      assert(user_sgpr_info.user_sgpr_idx == user_sgpr_info.num_sgpr);
+      unsigned idx = user_sgpr_info.user_sgpr_idx;
+      for (unsigned i = 0; i < 3; i++) {
+         if (ctx->program->info->cs.uses_block_id[i])
+            add_arg(&args, s1, &ctx->workgroup_ids[i], idx++);
+      }
+
+      if (ctx->program->info->cs.uses_local_invocation_idx)
+         add_arg(&args, s1, &ctx->tg_size, idx++);
+      if (ctx->scratch_enabled)
+         add_arg(&args, s1, &ctx->scratch_offset, idx++);
+
+      add_arg(&args, v1, &ctx->local_invocation_ids[0], vgpr_idx++);
+      add_arg(&args, v1, &ctx->local_invocation_ids[1], vgpr_idx++);
+      add_arg(&args, v1, &ctx->local_invocation_ids[2], vgpr_idx++);
+      break;
+   }
+   default:
+      unreachable("Shader stage not implemented");
+   }
+
+   ctx->program->info->num_input_vgprs = 0;
+   ctx->program->info->num_input_sgprs = args.num_sgprs_used;
+   ctx->program->info->num_user_sgprs = user_sgpr_info.num_sgpr;
+   ctx->program->info->num_input_vgprs = args.num_vgprs_used;
+
+   aco_ptr<Pseudo_instruction> startpgm{create_instruction<Pseudo_instruction>(aco_opcode::p_startpgm, Format::PSEUDO, 0, args.count + 1)};
+   for (unsigned i = 0; i < args.count; i++) {
+      if (args.assign[i]) {
+         *args.assign[i] = Temp{ctx->program->allocateId(), args.types[i]};
+         startpgm->definitions[i] = Definition(*args.assign[i]);
+         startpgm->definitions[i].setFixed(args.reg[i]);
+      }
+   }
+   startpgm->definitions[args.count] = Definition{ctx->program->allocateId(), exec, s2};
+   ctx->block->instructions.push_back(std::move(startpgm));
+}
+
+int
+type_size(const struct glsl_type *type, bool bindless)
+{
+   // TODO: don't we need type->std430_base_alignment() here?
+   return glsl_count_attribute_slots(type, false);
+}
+
+void
+shared_var_info(const struct glsl_type *type, unsigned *size, unsigned *align)
+{
+   assert(glsl_type_is_vector_or_scalar(type));
+
+   uint32_t comp_size = glsl_type_is_boolean(type)
+      ? 4 : glsl_get_bit_size(type) / 8;
+   unsigned length = glsl_get_vector_elements(type);
+   *size = comp_size * length,
+   *align = comp_size;
+}
+
+int
+get_align(nir_variable_mode mode, bool is_store, unsigned bit_size, unsigned num_components)
+{
+   /* TODO: ACO doesn't have good support for non-32-bit reads/writes yet */
+   if (bit_size != 32)
+      return -1;
+
+   switch (mode) {
+   case nir_var_mem_ubo:
+   case nir_var_mem_ssbo:
+   //case nir_var_mem_push_const: enable with 1240!
+   case nir_var_mem_shared:
+      /* TODO: what are the alignment requirements for LDS? */
+      return num_components <= 4 ? 4 : -1;
+   default:
+      return -1;
+   }
+}
+
+void
+setup_vs_variables(isel_context *ctx, nir_shader *nir)
+{
+   nir_foreach_variable(variable, &nir->inputs)
+   {
+      variable->data.driver_location = variable->data.location * 4;
+   }
+   nir_foreach_variable(variable, &nir->outputs)
+   {
+      variable->data.driver_location = variable->data.location * 4;
+   }
+
+   radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
+
+   memset(outinfo->vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
+          sizeof(outinfo->vs_output_param_offset));
+
+   ctx->needs_instance_id = ctx->program->info->vs.needs_instance_id;
+
+   bool export_clip_dists = ctx->options->key.vs_common_out.export_clip_dists;
+
+   outinfo->param_exports = 0;
+   int pos_written = 0x1;
+   if (outinfo->writes_pointsize || outinfo->writes_viewport_index || outinfo->writes_layer)
+      pos_written |= 1 << 1;
+
+   nir_foreach_variable(variable, &nir->outputs)
+   {
+      int idx = variable->data.location;
+      unsigned slots = variable->type->count_attribute_slots(false);
+      if (variable->data.compact) {
+         unsigned component_count = variable->data.location_frac + variable->type->length;
+         slots = (component_count + 3) / 4;
+      }
+
+      if (idx >= VARYING_SLOT_VAR0 || idx == VARYING_SLOT_LAYER || idx == VARYING_SLOT_PRIMITIVE_ID ||
+          ((idx == VARYING_SLOT_CLIP_DIST0 || idx == VARYING_SLOT_CLIP_DIST1) && export_clip_dists)) {
+         for (unsigned i = 0; i < slots; i++) {
+            if (outinfo->vs_output_param_offset[idx + i] == AC_EXP_PARAM_UNDEFINED)
+               outinfo->vs_output_param_offset[idx + i] = outinfo->param_exports++;
+         }
+      }
+   }
+   if (outinfo->writes_layer &&
+       outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] == AC_EXP_PARAM_UNDEFINED) {
+      /* when ctx->options->key.has_multiview_view_index = true, the layer
+       * variable isn't declared in NIR and it's isel's job to get the layer */
+      outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] = outinfo->param_exports++;
+   }
+
+   if (outinfo->export_prim_id) {
+      assert(outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] == AC_EXP_PARAM_UNDEFINED);
+      outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] = outinfo->param_exports++;
+   }
+
+   ctx->num_clip_distances = util_bitcount(outinfo->clip_dist_mask);
+   ctx->num_cull_distances = util_bitcount(outinfo->cull_dist_mask);
+
+   assert(ctx->num_clip_distances + ctx->num_cull_distances <= 8);
+
+   if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
+      pos_written |= 1 << 2;
+   if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
+      pos_written |= 1 << 3;
+
+   outinfo->pos_exports = util_bitcount(pos_written);
+}
+
+void
+setup_variables(isel_context *ctx, nir_shader *nir)
+{
+   switch (nir->info.stage) {
+   case MESA_SHADER_FRAGMENT: {
+      nir_foreach_variable(variable, &nir->outputs)
+      {
+         int idx = variable->data.location + variable->data.index;
+         variable->data.driver_location = idx * 4;
+      }
+      break;
+   }
+   case MESA_SHADER_COMPUTE: {
+      unsigned lds_allocation_size_unit = 4 * 64;
+      if (ctx->program->chip_class >= GFX7)
+         lds_allocation_size_unit = 4 * 128;
+      ctx->program->config->lds_size = (nir->info.cs.shared_size + lds_allocation_size_unit - 1) / lds_allocation_size_unit;
+      break;
+   }
+   case MESA_SHADER_VERTEX: {
+      setup_vs_variables(ctx, nir);
+      break;
+   }
+   default:
+      unreachable("Unhandled shader stage.");
+   }
+}
+
+isel_context
+setup_isel_context(Program* program,
+                   unsigned shader_count,
+                   struct nir_shader *const *shaders,
+                   ac_shader_config* config,
+                   radv_shader_info *info,
+                   radv_nir_compiler_options *options)
+{
+   program->stage = 0;
+   for (unsigned i = 0; i < shader_count; i++) {
+      switch (shaders[i]->info.stage) {
+      case MESA_SHADER_VERTEX:
+         program->stage |= sw_vs;
+         break;
+      case MESA_SHADER_TESS_CTRL:
+         program->stage |= sw_tcs;
+         break;
+      case MESA_SHADER_TESS_EVAL:
+         program->stage |= sw_tes;
+         break;
+      case MESA_SHADER_GEOMETRY:
+         program->stage |= sw_gs;
+         break;
+      case MESA_SHADER_FRAGMENT:
+         program->stage |= sw_fs;
+         break;
+      case MESA_SHADER_COMPUTE:
+         program->stage |= sw_cs;
+         break;
+      default:
+         unreachable("Shader stage not implemented");
+      }
+   }
+   if (program->stage == sw_vs)
+      program->stage |= hw_vs;
+   else if (program->stage == sw_fs)
+      program->stage |= hw_fs;
+   else if (program->stage == sw_cs)
+      program->stage |= hw_cs;
+   else
+      unreachable("Shader stage not implemented");
+
+   program->config = config;
+   program->info = info;
+   program->chip_class = options->chip_class;
+   program->family = options->family;
+   program->sgpr_limit = options->chip_class >= GFX8 ? 102 : 104;
+   if (options->family == CHIP_TONGA || options->family == CHIP_ICELAND)
+      program->sgpr_limit = 94; /* workaround hardware bug */
+
+   for (unsigned i = 0; i < MAX_SETS; ++i)
+      program->info->user_sgprs_locs.descriptor_sets[i].sgpr_idx = -1;
+   for (unsigned i = 0; i < AC_UD_MAX_UD; ++i)
+      program->info->user_sgprs_locs.shader_data[i].sgpr_idx = -1;
+
+   isel_context ctx = {};
+   ctx.program = program;
+   ctx.options = options;
+   ctx.stage = program->stage;
+
+   for (unsigned i = 0; i < fs_input::max_inputs; ++i)
+      ctx.fs_inputs[i] = Temp(0, v1);
+   ctx.fs_inputs[fs_input::persp_pull_model] = Temp(0, v3);
+   for (unsigned i = 0; i < MAX_SETS; ++i)
+      ctx.descriptor_sets[i] = Temp(0, s1);
+   for (unsigned i = 0; i < MAX_INLINE_PUSH_CONSTS; ++i)
+      ctx.inline_push_consts[i] = Temp(0, s1);
+   for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
+      for (unsigned j = 0; j < 4; ++j)
+         ctx.vs_output.outputs[i][j] = Temp(0, v1);
+   }
+
+   for (unsigned i = 0; i < shader_count; i++) {
+      nir_shader *nir = shaders[i];
+
+      /* align and copy constant data */
+      while (program->constant_data.size() % 4u)
+         program->constant_data.push_back(0);
+      ctx.constant_data_offset = program->constant_data.size();
+      program->constant_data.insert(program->constant_data.end(),
+                                    (uint8_t*)nir->constant_data,
+                                    (uint8_t*)nir->constant_data + nir->constant_data_size);
+
+      /* the variable setup has to be done before lower_io / CSE */
+      if (nir->info.stage == MESA_SHADER_COMPUTE)
+         nir_lower_vars_to_explicit_types(nir, nir_var_mem_shared, shared_var_info);
+      setup_variables(&ctx, nir);
+
+      /* optimize and lower memory operations */
+      bool lower_to_scalar = false;
+      bool lower_pack = false;
+      // TODO: uncomment this once !1240 is merged
+      /*if (nir_opt_load_store_vectorize(nir,
+                                       (nir_variable_mode)(nir_var_mem_ssbo | nir_var_mem_ubo |
+                                                           nir_var_mem_push_const | nir_var_mem_shared),
+                                       get_align)) {
+         lower_to_scalar = true;
+         lower_pack = true;
+      }*/
+      if (nir->info.stage == MESA_SHADER_COMPUTE)
+         lower_to_scalar |= nir_lower_explicit_io(nir, nir_var_mem_shared, nir_address_format_32bit_offset);
+      else
+         nir_lower_io(nir, (nir_variable_mode)(nir_var_shader_in | nir_var_shader_out), type_size, (nir_lower_io_options)0);
+      nir_lower_explicit_io(nir, nir_var_mem_global, nir_address_format_64bit_global);
+
+      if (lower_to_scalar)
+         nir_lower_alu_to_scalar(nir, NULL, NULL);
+      if (lower_pack)
+         nir_lower_pack(nir);
+
+      /* lower ALU operations */
+      nir_opt_idiv_const(nir, 32);
+      nir_lower_idiv(nir); // TODO: use the LLVM path once !1239 is merged
+
+      // TODO: implement logic64 in aco, it's more effective for sgprs
+      nir_lower_int64(nir, (nir_lower_int64_options) (nir_lower_imul64 |
+                                                      nir_lower_imul_high64 |
+                                                      nir_lower_imul_2x32_64 |
+                                                      nir_lower_divmod64 |
+                                                      nir_lower_logic64 |
+                                                      nir_lower_minmax64 |
+                                                      nir_lower_iabs64 |
+                                                      nir_lower_ineg64));
+
+      /* optimize the lowered ALU operations */
+      nir_copy_prop(nir);
+      nir_opt_constant_folding(nir);
+      nir_opt_algebraic(nir);
+      nir_opt_algebraic_late(nir);
+      nir_opt_constant_folding(nir);
+
+      /* cleanup passes */
+      nir_lower_load_const_to_scalar(nir);
+      nir_opt_cse(nir);
+      nir_opt_dce(nir);
+      nir_opt_shrink_load(nir);
+      nir_move_options move_opts = (nir_move_options)(
+         nir_move_const_undef | nir_move_load_ubo | nir_move_load_input | nir_move_comparisons);
+      //nir_opt_sink(nir, move_opts); // TODO: enable this once !1664 is merged
+      nir_opt_move(nir, move_opts);
+      nir_convert_to_lcssa(nir, true, false);
+      nir_lower_phis_to_scalar(nir);
+
+      nir_function_impl *func = nir_shader_get_entrypoint(nir);
+      nir_index_ssa_defs(func);
+
+      if (options->dump_preoptir) {
+         fprintf(stderr, "NIR shader before instruction selection:\n");
+         nir_print_shader(nir, stderr);
+      }
+   }
+
+   unsigned scratch_size = 0;
+   for (unsigned i = 0; i < shader_count; i++)
+      scratch_size = std::max(scratch_size, shaders[i]->scratch_size);
+   ctx.scratch_enabled = scratch_size > 0;
+   ctx.program->config->scratch_bytes_per_wave = align(scratch_size * ctx.options->wave_size, 1024);
+   ctx.program->config->float_mode = V_00B028_FP_64_DENORMS;
+   ctx.program->info->wave_size = ctx.options->wave_size;
+
+   ctx.block = ctx.program->create_and_insert_block();
+   ctx.block->loop_nest_depth = 0;
+   ctx.block->kind = block_kind_top_level;
+
+   return ctx;
+}
+
+}