glsl/nir: Produce unified atomics

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Reviewed-by: Emma Anholt <emma@anholt.net>
Reviewed-by: Jesse Natalie <jenatali@microsoft.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/23036>

4 files changed, 95 insertions, 124 deletions

diff --git a/src/compiler/glsl/gl_nir_link_uniforms.c b/src/compiler/glsl/gl_nir_link_uniforms.c
index f9721393fbc..83054469d3a 100644
--- a/src/compiler/glsl/gl_nir_link_uniforms.c
+++ b/src/compiler/glsl/gl_nir_link_uniforms.c
@@ -546,16 +546,8 @@ add_var_use_shader(nir_shader *shader, struct hash_table *live)
                   case nir_intrinsic_atomic_counter_comp_swap_deref:
                   case nir_intrinsic_image_deref_load:
                   case nir_intrinsic_image_deref_store:
-                  case nir_intrinsic_image_deref_atomic_add:
-                  case nir_intrinsic_image_deref_atomic_umin:
-                  case nir_intrinsic_image_deref_atomic_imin:
-                  case nir_intrinsic_image_deref_atomic_umax:
-                  case nir_intrinsic_image_deref_atomic_imax:
-                  case nir_intrinsic_image_deref_atomic_and:
-                  case nir_intrinsic_image_deref_atomic_or:
-                  case nir_intrinsic_image_deref_atomic_xor:
-                  case nir_intrinsic_image_deref_atomic_exchange:
-                  case nir_intrinsic_image_deref_atomic_comp_swap:
+                  case nir_intrinsic_image_deref_atomic:
+                  case nir_intrinsic_image_deref_atomic_swap:
                   case nir_intrinsic_image_deref_size:
                   case nir_intrinsic_image_deref_samples:
                   case nir_intrinsic_load_deref:
diff --git a/src/compiler/glsl/gl_nir_lower_images.c b/src/compiler/glsl/gl_nir_lower_images.c
index 9bf2129f4b7..4d2dd2b4ac3 100644
--- a/src/compiler/glsl/gl_nir_lower_images.c
+++ b/src/compiler/glsl/gl_nir_lower_images.c
@@ -64,19 +64,8 @@ lower_instr(nir_builder *b, nir_instr *instr, void *cb_data)
    nir_variable *var;
 
    switch (intrinsic->intrinsic) {
-   case nir_intrinsic_image_deref_atomic_add:
-   case nir_intrinsic_image_deref_atomic_imin:
-   case nir_intrinsic_image_deref_atomic_umin:
-   case nir_intrinsic_image_deref_atomic_imax:
-   case nir_intrinsic_image_deref_atomic_umax:
-   case nir_intrinsic_image_deref_atomic_and:
-   case nir_intrinsic_image_deref_atomic_or:
-   case nir_intrinsic_image_deref_atomic_xor:
-   case nir_intrinsic_image_deref_atomic_exchange:
-   case nir_intrinsic_image_deref_atomic_comp_swap:
-   case nir_intrinsic_image_deref_atomic_fadd:
-   case nir_intrinsic_image_deref_atomic_inc_wrap:
-   case nir_intrinsic_image_deref_atomic_dec_wrap:
+   case nir_intrinsic_image_deref_atomic:
+   case nir_intrinsic_image_deref_atomic_swap:
    case nir_intrinsic_image_deref_load:
    case nir_intrinsic_image_deref_samples:
    case nir_intrinsic_image_deref_size:
diff --git a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c
index 934369d74fe..7b529bc4347 100644
--- a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c
+++ b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c
@@ -316,17 +316,8 @@ lower_intrinsic(nir_intrinsic_instr *instr,
 {
    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
        instr->intrinsic == nir_intrinsic_image_deref_store ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_add ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_imin ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_umin ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_imax ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_umax ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_and ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_or ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_xor ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_exchange ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap ||
-       instr->intrinsic == nir_intrinsic_image_deref_atomic_fadd ||
+       instr->intrinsic == nir_intrinsic_image_deref_atomic ||
+       instr->intrinsic == nir_intrinsic_image_deref_atomic_swap ||
        instr->intrinsic == nir_intrinsic_image_deref_size ||
        instr->intrinsic == nir_intrinsic_image_deref_samples_identical ||
        instr->intrinsic == nir_intrinsic_image_deref_descriptor_amd ||
diff --git a/src/compiler/glsl/glsl_to_nir.cpp b/src/compiler/glsl/glsl_to_nir.cpp
index 00b6afa1a20..3b2ec155bf6 100644
--- a/src/compiler/glsl/glsl_to_nir.cpp
+++ b/src/compiler/glsl/glsl_to_nir.cpp
@@ -970,53 +970,64 @@ nir_visitor::visit(ir_call *ir)
    if (ir->callee->is_intrinsic()) {
       nir_intrinsic_op op;
 
+      /* Initialize to something because gcc complains otherwise */
+      nir_atomic_op atomic_op = nir_atomic_op_iadd;
+
       switch (ir->callee->intrinsic_id) {
       case ir_intrinsic_generic_atomic_add:
-         op = ir->return_deref->type->is_integer_32_64()
-            ? nir_intrinsic_deref_atomic_add : nir_intrinsic_deref_atomic_fadd;
+         op = nir_intrinsic_deref_atomic;
+         atomic_op = ir->return_deref->type->is_integer_32_64()
+            ? nir_atomic_op_iadd : nir_atomic_op_fadd;
          break;
       case ir_intrinsic_generic_atomic_and:
-         op = nir_intrinsic_deref_atomic_and;
+         op = nir_intrinsic_deref_atomic;
+         atomic_op = nir_atomic_op_iand;
          break;
       case ir_intrinsic_generic_atomic_or:
-         op = nir_intrinsic_deref_atomic_or;
+         op = nir_intrinsic_deref_atomic;
+         atomic_op = nir_atomic_op_ior;
          break;
       case ir_intrinsic_generic_atomic_xor:
-         op = nir_intrinsic_deref_atomic_xor;
+         op = nir_intrinsic_deref_atomic;
+         atomic_op = nir_atomic_op_ixor;
          break;
       case ir_intrinsic_generic_atomic_min:
          assert(ir->return_deref);
+         op = nir_intrinsic_deref_atomic;
          if (ir->return_deref->type == glsl_type::int_type ||
              ir->return_deref->type == glsl_type::int64_t_type)
-            op = nir_intrinsic_deref_atomic_imin;
+             atomic_op = nir_atomic_op_imin;
          else if (ir->return_deref->type == glsl_type::uint_type ||
                   ir->return_deref->type == glsl_type::uint64_t_type)
-            op = nir_intrinsic_deref_atomic_umin;
+             atomic_op = nir_atomic_op_umin;
          else if (ir->return_deref->type == glsl_type::float_type)
-            op = nir_intrinsic_deref_atomic_fmin;
+             atomic_op = nir_atomic_op_fmin;
          else
             unreachable("Invalid type");
          break;
       case ir_intrinsic_generic_atomic_max:
          assert(ir->return_deref);
+         op = nir_intrinsic_deref_atomic;
          if (ir->return_deref->type == glsl_type::int_type ||
              ir->return_deref->type == glsl_type::int64_t_type)
-            op = nir_intrinsic_deref_atomic_imax;
+             atomic_op = nir_atomic_op_imax;
          else if (ir->return_deref->type == glsl_type::uint_type ||
                   ir->return_deref->type == glsl_type::uint64_t_type)
-            op = nir_intrinsic_deref_atomic_umax;
+             atomic_op = nir_atomic_op_umax;
          else if (ir->return_deref->type == glsl_type::float_type)
-            op = nir_intrinsic_deref_atomic_fmax;
+             atomic_op = nir_atomic_op_fmax;
          else
             unreachable("Invalid type");
          break;
       case ir_intrinsic_generic_atomic_exchange:
-         op = nir_intrinsic_deref_atomic_exchange;
+         op = nir_intrinsic_deref_atomic;
+         atomic_op = nir_atomic_op_xchg;
          break;
       case ir_intrinsic_generic_atomic_comp_swap:
-         op = ir->return_deref->type->is_integer_32_64()
-            ? nir_intrinsic_deref_atomic_comp_swap
-            : nir_intrinsic_deref_atomic_fcomp_swap;
+         op = nir_intrinsic_deref_atomic_swap;
+         atomic_op = ir->return_deref->type->is_integer_32_64()
+            ? nir_atomic_op_cmpxchg
+            : nir_atomic_op_fcmpxchg;
          break;
       case ir_intrinsic_atomic_counter_read:
          op = nir_intrinsic_atomic_counter_read_deref;
@@ -1058,46 +1069,56 @@ nir_visitor::visit(ir_call *ir)
          op = nir_intrinsic_image_deref_store;
          break;
       case ir_intrinsic_image_atomic_add:
-         op = ir->return_deref->type->is_integer_32_64()
-            ? nir_intrinsic_image_deref_atomic_add
-            : nir_intrinsic_image_deref_atomic_fadd;
+         op = nir_intrinsic_image_deref_atomic;
+         atomic_op = ir->return_deref->type->is_integer_32_64()
+            ? nir_atomic_op_iadd
+            : nir_atomic_op_fadd;
          break;
       case ir_intrinsic_image_atomic_min:
+         op = nir_intrinsic_image_deref_atomic;
          if (ir->return_deref->type == glsl_type::int_type)
-            op = nir_intrinsic_image_deref_atomic_imin;
+            atomic_op = nir_atomic_op_imin;
          else if (ir->return_deref->type == glsl_type::uint_type)
-            op = nir_intrinsic_image_deref_atomic_umin;
+            atomic_op = nir_atomic_op_umin;
          else
             unreachable("Invalid type");
          break;
       case ir_intrinsic_image_atomic_max:
+         op = nir_intrinsic_image_deref_atomic;
          if (ir->return_deref->type == glsl_type::int_type)
-            op = nir_intrinsic_image_deref_atomic_imax;
+            atomic_op = nir_atomic_op_imax;
          else if (ir->return_deref->type == glsl_type::uint_type)
-            op = nir_intrinsic_image_deref_atomic_umax;
+            atomic_op = nir_atomic_op_umax;
          else
             unreachable("Invalid type");
          break;
       case ir_intrinsic_image_atomic_and:
-         op = nir_intrinsic_image_deref_atomic_and;
+         op = nir_intrinsic_image_deref_atomic;
+         atomic_op = nir_atomic_op_iand;
          break;
       case ir_intrinsic_image_atomic_or:
-         op = nir_intrinsic_image_deref_atomic_or;
+         op = nir_intrinsic_image_deref_atomic;
+         atomic_op = nir_atomic_op_ior;
          break;
       case ir_intrinsic_image_atomic_xor:
-         op = nir_intrinsic_image_deref_atomic_xor;
+         op = nir_intrinsic_image_deref_atomic;
+         atomic_op = nir_atomic_op_ixor;
          break;
       case ir_intrinsic_image_atomic_exchange:
-         op = nir_intrinsic_image_deref_atomic_exchange;
+         op = nir_intrinsic_image_deref_atomic;
+         atomic_op = nir_atomic_op_xchg;
          break;
       case ir_intrinsic_image_atomic_comp_swap:
-         op = nir_intrinsic_image_deref_atomic_comp_swap;
+         op = nir_intrinsic_image_deref_atomic_swap;
+         atomic_op = nir_atomic_op_cmpxchg;
          break;
       case ir_intrinsic_image_atomic_inc_wrap:
-         op = nir_intrinsic_image_deref_atomic_inc_wrap;
+         op = nir_intrinsic_image_deref_atomic;
+         atomic_op = nir_atomic_op_inc_wrap;
          break;
       case ir_intrinsic_image_atomic_dec_wrap:
-         op = nir_intrinsic_image_deref_atomic_dec_wrap;
+         op = nir_intrinsic_image_deref_atomic;
+         atomic_op = nir_atomic_op_dec_wrap;
          break;
       case ir_intrinsic_memory_barrier:
          op = shader->options->use_scoped_barrier
@@ -1154,52 +1175,60 @@ nir_visitor::visit(ir_call *ir)
          op = nir_intrinsic_store_shared;
          break;
       case ir_intrinsic_shared_atomic_add:
-         op = ir->return_deref->type->is_integer_32_64()
-            ? nir_intrinsic_shared_atomic_add
-            : nir_intrinsic_shared_atomic_fadd;
+         op = nir_intrinsic_shared_atomic;
+         atomic_op = ir->return_deref->type->is_integer_32_64()
+            ? nir_atomic_op_iadd
+            : nir_atomic_op_fadd;
          break;
       case ir_intrinsic_shared_atomic_and:
-         op = nir_intrinsic_shared_atomic_and;
+         op = nir_intrinsic_shared_atomic;
+         atomic_op = nir_atomic_op_iand;
          break;
       case ir_intrinsic_shared_atomic_or:
-         op = nir_intrinsic_shared_atomic_or;
+         op = nir_intrinsic_shared_atomic;
+         atomic_op = nir_atomic_op_ior;
          break;
       case ir_intrinsic_shared_atomic_xor:
-         op = nir_intrinsic_shared_atomic_xor;
+         op = nir_intrinsic_shared_atomic;
+         atomic_op = nir_atomic_op_ixor;
          break;
       case ir_intrinsic_shared_atomic_min:
          assert(ir->return_deref);
+         op = nir_intrinsic_shared_atomic;
          if (ir->return_deref->type == glsl_type::int_type ||
              ir->return_deref->type == glsl_type::int64_t_type)
-            op = nir_intrinsic_shared_atomic_imin;
+            atomic_op = nir_atomic_op_imin;
          else if (ir->return_deref->type == glsl_type::uint_type ||
                   ir->return_deref->type == glsl_type::uint64_t_type)
-            op = nir_intrinsic_shared_atomic_umin;
+            atomic_op = nir_atomic_op_umin;
          else if (ir->return_deref->type == glsl_type::float_type)
-            op = nir_intrinsic_shared_atomic_fmin;
+            atomic_op = nir_atomic_op_fmin;
          else
             unreachable("Invalid type");
          break;
       case ir_intrinsic_shared_atomic_max:
          assert(ir->return_deref);
+         op = nir_intrinsic_shared_atomic;
          if (ir->return_deref->type == glsl_type::int_type ||
              ir->return_deref->type == glsl_type::int64_t_type)
-            op = nir_intrinsic_shared_atomic_imax;
+            atomic_op = nir_atomic_op_imax;
          else if (ir->return_deref->type == glsl_type::uint_type ||
                   ir->return_deref->type == glsl_type::uint64_t_type)
-            op = nir_intrinsic_shared_atomic_umax;
+            atomic_op = nir_atomic_op_umax;
          else if (ir->return_deref->type == glsl_type::float_type)
-            op = nir_intrinsic_shared_atomic_fmax;
+            atomic_op = nir_atomic_op_fmax;
          else
             unreachable("Invalid type");
          break;
       case ir_intrinsic_shared_atomic_exchange:
-         op = nir_intrinsic_shared_atomic_exchange;
+         op = nir_intrinsic_shared_atomic;
+         atomic_op = nir_atomic_op_xchg;
          break;
       case ir_intrinsic_shared_atomic_comp_swap:
-         op = ir->return_deref->type->is_integer_32_64()
-            ? nir_intrinsic_shared_atomic_comp_swap
-            : nir_intrinsic_shared_atomic_fcomp_swap;
+         op = nir_intrinsic_shared_atomic_swap;
+         atomic_op = ir->return_deref->type->is_integer_32_64()
+            ? nir_atomic_op_cmpxchg
+            : nir_atomic_op_fcmpxchg;
          break;
       case ir_intrinsic_vote_any:
          op = nir_intrinsic_vote_any;
@@ -1233,20 +1262,8 @@ nir_visitor::visit(ir_call *ir)
       nir_ssa_def *ret = &instr->dest.ssa;
 
       switch (op) {
-      case nir_intrinsic_deref_atomic_add:
-      case nir_intrinsic_deref_atomic_imin:
-      case nir_intrinsic_deref_atomic_umin:
-      case nir_intrinsic_deref_atomic_imax:
-      case nir_intrinsic_deref_atomic_umax:
-      case nir_intrinsic_deref_atomic_and:
-      case nir_intrinsic_deref_atomic_or:
-      case nir_intrinsic_deref_atomic_xor:
-      case nir_intrinsic_deref_atomic_exchange:
-      case nir_intrinsic_deref_atomic_comp_swap:
-      case nir_intrinsic_deref_atomic_fadd:
-      case nir_intrinsic_deref_atomic_fmin:
-      case nir_intrinsic_deref_atomic_fmax:
-      case nir_intrinsic_deref_atomic_fcomp_swap: {
+      case nir_intrinsic_deref_atomic:
+      case nir_intrinsic_deref_atomic_swap: {
          int param_count = ir->actual_parameters.length();
          assert(param_count == 2 || param_count == 3);
 
@@ -1269,6 +1286,7 @@ nir_visitor::visit(ir_call *ir)
          }
          instr->src[0] = nir_src_for_ssa(&nir_deref->dest.ssa);
 
+         nir_intrinsic_set_atomic_op(instr, atomic_op);
          nir_intrinsic_set_access(instr, deref_get_qualifier(nir_deref));
 
          /* data1 parameter (this is always present) */
@@ -1278,8 +1296,7 @@ nir_visitor::visit(ir_call *ir)
 
          /* data2 parameter (only with atomic_comp_swap) */
          if (param_count == 3) {
-            assert(op == nir_intrinsic_deref_atomic_comp_swap ||
-                   op == nir_intrinsic_deref_atomic_fcomp_swap);
+            assert(op == nir_intrinsic_deref_atomic_swap);
             param = param->get_next();
             inst = (ir_instruction *) param;
             instr->src[2] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
@@ -1338,21 +1355,10 @@ nir_visitor::visit(ir_call *ir)
       }
       case nir_intrinsic_image_deref_load:
       case nir_intrinsic_image_deref_store:
-      case nir_intrinsic_image_deref_atomic_add:
-      case nir_intrinsic_image_deref_atomic_imin:
-      case nir_intrinsic_image_deref_atomic_umin:
-      case nir_intrinsic_image_deref_atomic_imax:
-      case nir_intrinsic_image_deref_atomic_umax:
-      case nir_intrinsic_image_deref_atomic_and:
-      case nir_intrinsic_image_deref_atomic_or:
-      case nir_intrinsic_image_deref_atomic_xor:
-      case nir_intrinsic_image_deref_atomic_exchange:
-      case nir_intrinsic_image_deref_atomic_comp_swap:
-      case nir_intrinsic_image_deref_atomic_fadd:
+      case nir_intrinsic_image_deref_atomic:
+      case nir_intrinsic_image_deref_atomic_swap:
       case nir_intrinsic_image_deref_samples:
       case nir_intrinsic_image_deref_size:
-      case nir_intrinsic_image_deref_atomic_inc_wrap:
-      case nir_intrinsic_image_deref_atomic_dec_wrap:
       case nir_intrinsic_image_deref_sparse_load: {
          /* Set the image variable dereference. */
          exec_node *param = ir->actual_parameters.get_head();
@@ -1362,6 +1368,11 @@ nir_visitor::visit(ir_call *ir)
 
          nir_intrinsic_set_access(instr, deref_get_qualifier(deref));
 
+         if (op == nir_intrinsic_image_deref_atomic ||
+             op == nir_intrinsic_image_deref_atomic_swap) {
+            nir_intrinsic_set_atomic_op(instr, atomic_op);
+         }
+
          instr->src[0] = nir_src_for_ssa(&deref->dest.ssa);
          param = param->get_next();
          nir_intrinsic_set_image_dim(instr,
@@ -1610,20 +1621,8 @@ nir_visitor::visit(ir_call *ir)
          nir_builder_instr_insert(&b, &instr->instr);
          break;
       }
-      case nir_intrinsic_shared_atomic_add:
-      case nir_intrinsic_shared_atomic_imin:
-      case nir_intrinsic_shared_atomic_umin:
-      case nir_intrinsic_shared_atomic_imax:
-      case nir_intrinsic_shared_atomic_umax:
-      case nir_intrinsic_shared_atomic_and:
-      case nir_intrinsic_shared_atomic_or:
-      case nir_intrinsic_shared_atomic_xor:
-      case nir_intrinsic_shared_atomic_exchange:
-      case nir_intrinsic_shared_atomic_comp_swap:
-      case nir_intrinsic_shared_atomic_fadd:
-      case nir_intrinsic_shared_atomic_fmin:
-      case nir_intrinsic_shared_atomic_fmax:
-      case nir_intrinsic_shared_atomic_fcomp_swap:  {
+      case nir_intrinsic_shared_atomic:
+      case nir_intrinsic_shared_atomic_swap: {
          int param_count = ir->actual_parameters.length();
          assert(param_count == 2 || param_count == 3);
 
@@ -1639,8 +1638,7 @@ nir_visitor::visit(ir_call *ir)
 
          /* data2 parameter (only with atomic_comp_swap) */
          if (param_count == 3) {
-            assert(op == nir_intrinsic_shared_atomic_comp_swap ||
-                   op == nir_intrinsic_shared_atomic_fcomp_swap);
+            assert(op == nir_intrinsic_shared_atomic_swap);
             param = param->get_next();
             inst = (ir_instruction *) param;
             instr->src[2] =
@@ -1653,6 +1651,7 @@ nir_visitor::visit(ir_call *ir)
          nir_ssa_dest_init(&instr->instr, &instr->dest,
                            ir->return_deref->type->vector_elements,
                            bit_size, NULL);
+         nir_intrinsic_set_atomic_op(instr, atomic_op);
          nir_builder_instr_insert(&b, &instr->instr);
          break;
       }