r600: Use unified atomics

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Reviewed-by: Gert Wollny <gert.wollny@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/23026>

4 files changed, 61 insertions, 134 deletions

diff --git a/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp b/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp
index c8982e0d65e..60892cbdbfb 100644
--- a/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp
@@ -443,88 +443,60 @@ RatInstr::do_print(std::ostream& os) const
 }
 
 static RatInstr::ERatOp
-get_rat_opcode(const nir_intrinsic_op opcode, pipe_format format)
+get_rat_opcode(const nir_atomic_op opcode)
 {
    switch (opcode) {
-   case nir_intrinsic_image_load:
-      return RatInstr::NOP_RTN;
-   case nir_intrinsic_ssbo_atomic_add:
-   case nir_intrinsic_image_atomic_add:
+   case nir_atomic_op_iadd:
       return RatInstr::ADD_RTN;
-   case nir_intrinsic_ssbo_atomic_and:
-   case nir_intrinsic_image_atomic_and:
+   case nir_atomic_op_iand:
       return RatInstr::AND_RTN;
-   case nir_intrinsic_ssbo_atomic_or:
-   case nir_intrinsic_image_atomic_or:
+   case nir_atomic_op_ior:
       return RatInstr::OR_RTN;
-   case nir_intrinsic_ssbo_atomic_imin:
-   case nir_intrinsic_image_atomic_imin:
+   case nir_atomic_op_imin:
       return RatInstr::MIN_INT_RTN;
-   case nir_intrinsic_ssbo_atomic_imax:
-   case nir_intrinsic_image_atomic_imax:
+   case nir_atomic_op_imax:
       return RatInstr::MAX_INT_RTN;
-   case nir_intrinsic_ssbo_atomic_umin:
-   case nir_intrinsic_image_atomic_umin:
+   case nir_atomic_op_umin:
       return RatInstr::MIN_UINT_RTN;
-   case nir_intrinsic_ssbo_atomic_umax:
-   case nir_intrinsic_image_atomic_umax:
+   case nir_atomic_op_umax:
       return RatInstr::MAX_UINT_RTN;
-   case nir_intrinsic_ssbo_atomic_xor:
-   case nir_intrinsic_image_atomic_xor:
+   case nir_atomic_op_ixor:
       return RatInstr::XOR_RTN;
-   case nir_intrinsic_ssbo_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_comp_swap:
-      if (util_format_is_float(format))
-         return RatInstr::CMPXCHG_FLT_RTN;
-      else
-         return RatInstr::CMPXCHG_INT_RTN;
-   case nir_intrinsic_ssbo_atomic_exchange:
-   case nir_intrinsic_image_atomic_exchange:
+   case nir_atomic_op_cmpxchg:
+      return RatInstr::CMPXCHG_INT_RTN;
+   case nir_atomic_op_xchg:
       return RatInstr::XCHG_RTN;
    default:
-      unreachable("Unsupported WO RAT instruction");
+      unreachable("Unsupported atomic");
    }
 }
 
 static RatInstr::ERatOp
-get_rat_opcode_wo(const nir_intrinsic_op opcode, pipe_format format)
+get_rat_opcode_wo(const nir_atomic_op opcode)
 {
    switch (opcode) {
-   case nir_intrinsic_ssbo_atomic_add:
-   case nir_intrinsic_image_atomic_add:
+   case nir_atomic_op_iadd:
       return RatInstr::ADD;
-   case nir_intrinsic_ssbo_atomic_and:
-   case nir_intrinsic_image_atomic_and:
+   case nir_atomic_op_iand:
       return RatInstr::AND;
-   case nir_intrinsic_ssbo_atomic_or:
-   case nir_intrinsic_image_atomic_or:
+   case nir_atomic_op_ior:
       return RatInstr::OR;
-   case nir_intrinsic_ssbo_atomic_imin:
-   case nir_intrinsic_image_atomic_imin:
+   case nir_atomic_op_imin:
       return RatInstr::MIN_INT;
-   case nir_intrinsic_ssbo_atomic_imax:
-   case nir_intrinsic_image_atomic_imax:
+   case nir_atomic_op_imax:
       return RatInstr::MAX_INT;
-   case nir_intrinsic_ssbo_atomic_umin:
-   case nir_intrinsic_image_atomic_umin:
+   case nir_atomic_op_umin:
       return RatInstr::MIN_UINT;
-   case nir_intrinsic_ssbo_atomic_umax:
-   case nir_intrinsic_image_atomic_umax:
+   case nir_atomic_op_umax:
       return RatInstr::MAX_UINT;
-   case nir_intrinsic_ssbo_atomic_xor:
-   case nir_intrinsic_image_atomic_xor:
+   case nir_atomic_op_ixor:
       return RatInstr::XOR;
-   case nir_intrinsic_ssbo_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_comp_swap:
-      if (util_format_is_float(format))
-         return RatInstr::CMPXCHG_FLT;
-      else
-         return RatInstr::CMPXCHG_INT;
-   case nir_intrinsic_ssbo_atomic_exchange:
-   case nir_intrinsic_image_atomic_exchange:
+   case nir_atomic_op_cmpxchg:
+      return RatInstr::CMPXCHG_INT;
+   case nir_atomic_op_xchg:
       return RatInstr::XCHG_RTN;
    default:
-      unreachable("Unsupported WO RAT instruction");
+      unreachable("Unsupported atomic");
    }
 }
 
@@ -536,30 +508,14 @@ RatInstr::emit(nir_intrinsic_instr *intr, Shader& shader)
       return emit_ssbo_load(intr, shader);
    case nir_intrinsic_store_ssbo:
       return emit_ssbo_store(intr, shader);
-   case nir_intrinsic_ssbo_atomic_add:
-   case nir_intrinsic_ssbo_atomic_comp_swap:
-   case nir_intrinsic_ssbo_atomic_or:
-   case nir_intrinsic_ssbo_atomic_xor:
-   case nir_intrinsic_ssbo_atomic_imax:
-   case nir_intrinsic_ssbo_atomic_imin:
-   case nir_intrinsic_ssbo_atomic_umax:
-   case nir_intrinsic_ssbo_atomic_umin:
-   case nir_intrinsic_ssbo_atomic_and:
-   case nir_intrinsic_ssbo_atomic_exchange:
+   case nir_intrinsic_ssbo_atomic:
+   case nir_intrinsic_ssbo_atomic_swap:
       return emit_ssbo_atomic_op(intr, shader);
    case nir_intrinsic_image_store:
       return emit_image_store(intr, shader);
    case nir_intrinsic_image_load:
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_image_atomic_imax:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
       return emit_image_load_or_atomic(intr, shader);
    case nir_intrinsic_image_size:
       return emit_image_size(intr, shader);
@@ -662,8 +618,8 @@ RatInstr::emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader)
    }
 
    bool read_result = !intr->dest.is_ssa || !list_is_empty(&intr->dest.ssa.uses);
-   auto opcode = read_result ? get_rat_opcode(intr->intrinsic, PIPE_FORMAT_R32_UINT)
-                             : get_rat_opcode_wo(intr->intrinsic, PIPE_FORMAT_R32_UINT);
+   auto opcode = read_result ? get_rat_opcode(nir_intrinsic_atomic_op(intr))
+                             : get_rat_opcode_wo(nir_intrinsic_atomic_op(intr));
 
    auto coord_orig = vf.src(intr->src[1], 0);
    auto coord = vf.temp_register(0);
@@ -676,7 +632,7 @@ RatInstr::emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader)
    shader.emit_instruction(
       new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write));
 
-   if (intr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) {
+   if (intr->intrinsic == nir_intrinsic_ssbo_atomic_swap) {
       shader.emit_instruction(
          new AluInstr(op1_mov, data_vec4[0], vf.src(intr->src[3], 0), AluInstr::write));
       shader.emit_instruction(
@@ -799,8 +755,10 @@ RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& shader)
    }
 
    bool read_result = !intrin->dest.is_ssa || !list_is_empty(&intrin->dest.ssa.uses);
-   auto opcode = read_result ? get_rat_opcode(intrin->intrinsic, PIPE_FORMAT_R32_UINT)
-                             : get_rat_opcode_wo(intrin->intrinsic, PIPE_FORMAT_R32_UINT);
+   bool image_load = (intrin->intrinsic == nir_intrinsic_image_load);
+   auto opcode = image_load  ? RatInstr::NOP_RTN :
+                 read_result ? get_rat_opcode(nir_intrinsic_atomic_op(intrin))
+                             : get_rat_opcode_wo(nir_intrinsic_atomic_op(intrin));
 
    auto coord_orig = vf.src_vec4(intrin->src[1], pin_chan);
    auto coord = vf.temp_vec4(pin_chgr);
@@ -821,7 +779,7 @@ RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& shader)
    shader.emit_instruction(
       new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write));
 
-   if (intrin->intrinsic == nir_intrinsic_image_atomic_comp_swap) {
+   if (intrin->intrinsic == nir_intrinsic_image_atomic_swap) {
       shader.emit_instruction(
          new AluInstr(op1_mov, data_vec4[0], vf.src(intrin->src[4], 0), AluInstr::write));
       shader.emit_instruction(
diff --git a/src/gallium/drivers/r600/sfn/sfn_nir.cpp b/src/gallium/drivers/r600/sfn/sfn_nir.cpp
index f35a62f1bf0..6d007fb2385 100644
--- a/src/gallium/drivers/r600/sfn/sfn_nir.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_nir.cpp
@@ -782,6 +782,7 @@ r600_finalize_nir(pipe_screen *screen, void *shader)
    NIR_PASS_V(nir, r600_nir_lower_pack_unpack_2x16);
 
    NIR_PASS_V(nir, r600_lower_shared_io);
+   NIR_PASS_V(nir, nir_lower_legacy_atomics);
    NIR_PASS_V(nir, r600_nir_lower_atomics);
 
    if (rs->b.gfx_level == CAYMAN)
diff --git a/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp b/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp
index 65ef7e1f8c1..1e6f0fe6a52 100644
--- a/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp
@@ -168,16 +168,8 @@ r600_legalize_image_load_store_filter(const nir_instr *instr, UNUSED const void
    switch (ir->intrinsic) {
    case nir_intrinsic_image_store:
    case nir_intrinsic_image_load:
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_image_atomic_imax:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
    case nir_intrinsic_image_size:
       return true;
    default:
diff --git a/src/gallium/drivers/r600/sfn/sfn_shader.cpp b/src/gallium/drivers/r600/sfn/sfn_shader.cpp
index df13abc6239..a8a8a63a3d7 100644
--- a/src/gallium/drivers/r600/sfn/sfn_shader.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_shader.cpp
@@ -663,27 +663,11 @@ Shader::scan_instruction(nir_instr *instr)
 
    // handle unhandled instructions
    switch (intr->intrinsic) {
-   case nir_intrinsic_ssbo_atomic_add:
-   case nir_intrinsic_ssbo_atomic_comp_swap:
-   case nir_intrinsic_ssbo_atomic_or:
-   case nir_intrinsic_ssbo_atomic_xor:
-   case nir_intrinsic_ssbo_atomic_imax:
-   case nir_intrinsic_ssbo_atomic_imin:
-   case nir_intrinsic_ssbo_atomic_umax:
-   case nir_intrinsic_ssbo_atomic_umin:
-   case nir_intrinsic_ssbo_atomic_and:
-   case nir_intrinsic_ssbo_atomic_exchange:
+   case nir_intrinsic_ssbo_atomic:
+   case nir_intrinsic_ssbo_atomic_swap:
    case nir_intrinsic_image_load:
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_image_atomic_imax:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
       m_flags.set(sh_needs_sbo_ret_address);
       FALLTHROUGH;
    case nir_intrinsic_image_store:
@@ -907,16 +891,8 @@ Shader::process_intrinsic(nir_intrinsic_instr *intr)
    case nir_intrinsic_memory_barrier:
       return emit_wait_ack();
 
-   case nir_intrinsic_shared_atomic_add:
-   case nir_intrinsic_shared_atomic_and:
-   case nir_intrinsic_shared_atomic_or:
-   case nir_intrinsic_shared_atomic_imax:
-   case nir_intrinsic_shared_atomic_umax:
-   case nir_intrinsic_shared_atomic_imin:
-   case nir_intrinsic_shared_atomic_umin:
-   case nir_intrinsic_shared_atomic_xor:
-   case nir_intrinsic_shared_atomic_exchange:
-   case nir_intrinsic_shared_atomic_comp_swap:
+   case nir_intrinsic_shared_atomic:
+   case nir_intrinsic_shared_atomic_swap:
       return emit_atomic_local_shared(intr);
    case nir_intrinsic_shader_clock:
       return emit_shader_clock(intr);
@@ -927,31 +903,31 @@ Shader::process_intrinsic(nir_intrinsic_instr *intr)
 }
 
 static ESDOp
-lds_op_from_intrinsic(nir_intrinsic_op op, bool ret)
+lds_op_from_intrinsic(nir_atomic_op op, bool ret)
 {
    switch (op) {
-   case nir_intrinsic_shared_atomic_add:
+   case nir_atomic_op_iadd:
       return ret ? LDS_ADD_RET : LDS_ADD;
-   case nir_intrinsic_shared_atomic_and:
+   case nir_atomic_op_iand:
       return ret ? LDS_AND_RET : LDS_AND;
-   case nir_intrinsic_shared_atomic_or:
+   case nir_atomic_op_ior:
       return ret ? LDS_OR_RET : LDS_OR;
-   case nir_intrinsic_shared_atomic_imax:
+   case nir_atomic_op_imax:
       return ret ? LDS_MAX_INT_RET : LDS_MAX_INT;
-   case nir_intrinsic_shared_atomic_umax:
+   case nir_atomic_op_umax:
       return ret ? LDS_MAX_UINT_RET : LDS_MAX_UINT;
-   case nir_intrinsic_shared_atomic_imin:
+   case nir_atomic_op_imin:
       return ret ? LDS_MIN_INT_RET : LDS_MIN_INT;
-   case nir_intrinsic_shared_atomic_umin:
+   case nir_atomic_op_umin:
       return ret ? LDS_MIN_UINT_RET : LDS_MIN_UINT;
-   case nir_intrinsic_shared_atomic_xor:
+   case nir_atomic_op_ixor:
       return ret ? LDS_XOR_RET : LDS_XOR;
-   case nir_intrinsic_shared_atomic_exchange:
+   case nir_atomic_op_xchg:
       return LDS_XCHG_RET;
-   case nir_intrinsic_shared_atomic_comp_swap:
+   case nir_atomic_op_cmpxchg:
       return LDS_CMP_XCHG_RET;
    default:
-      unreachable("Unsupported shared atomic opcode");
+      unreachable("Unsupported shared atomic_op opcode");
    }
 }
 
@@ -977,14 +953,14 @@ Shader::emit_atomic_local_shared(nir_intrinsic_instr *instr)
 
    auto dest_value = uses_retval ? vf.dest(instr->dest, 0, pin_free) : nullptr;
 
-   auto op = lds_op_from_intrinsic(instr->intrinsic, uses_retval);
+   auto op = lds_op_from_intrinsic(nir_intrinsic_atomic_op(instr), uses_retval);
 
    auto address = vf.src(instr->src[0], 0);
 
    AluInstr::SrcValues src;
    src.push_back(vf.src(instr->src[1], 0));
 
-   if (unlikely(instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap))
+   if (unlikely(instr->intrinsic == nir_intrinsic_shared_atomic_swap))
       src.push_back(vf.src(instr->src[2], 0));
    emit_instruction(new LDSAtomicInstr(op, dest_value, address, src));
    return true;