ir3: Use unified atomics

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Reviewed-by: Jesse Natalie <jenatali@microsoft.com>
Reviewed-by: Rob Clark <robclark@freedesktop.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22914>

7 files changed, 148 insertions, 318 deletions

diff --git a/src/freedreno/ir3/ir3_a4xx.c b/src/freedreno/ir3/ir3_a4xx.c
index 5a87078aa11..3c3489bb0f6 100644
--- a/src/freedreno/ir3/ir3_a4xx.c
+++ b/src/freedreno/ir3/ir3_a4xx.c
@@ -126,6 +126,40 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    array_insert(b, b->keeps, stgb);
 }
 
+static struct ir3_instruction *
+emit_atomic(struct ir3_block *b,
+            nir_atomic_op op,
+            struct ir3_instruction *bo,
+            struct ir3_instruction *data,
+            struct ir3_instruction *offset,
+            struct ir3_instruction *byte_offset)
+{
+   switch (op) {
+   case nir_atomic_op_iadd:
+      return ir3_ATOMIC_S_ADD(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_imin:
+      return ir3_ATOMIC_S_MIN(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_umin:
+      return ir3_ATOMIC_S_MIN(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_imax:
+      return ir3_ATOMIC_S_MAX(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_umax:
+      return ir3_ATOMIC_S_MAX(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_iand:
+      return ir3_ATOMIC_S_AND(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_ior:
+      return ir3_ATOMIC_S_OR(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_ixor:
+      return ir3_ATOMIC_S_XOR(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_xchg:
+      return ir3_ATOMIC_S_XCHG(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   case nir_atomic_op_cmpxchg:
+      return ir3_ATOMIC_S_CMPXCHG(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
+   default:
+      unreachable("boo");
+   }
+}
+
 /*
  * SSBO atomic intrinsics
  *
@@ -140,7 +174,7 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
  * 1: The byte offset into the SSBO buffer of the variable that the atomic
  *    operation will operate on.
  * 2: The data parameter to the atomic function (i.e. the value to add
- *    in ssbo_atomic_add, etc).
+ *    in, etc).
  * 3: CompSwap: the second data parameter.
  *    Non-CompSwap: The dword offset into the SSBO buffer variable.
  * 4: CompSwap: The dword offset into the SSBO buffer variable.
@@ -152,8 +186,8 @@ static struct ir3_instruction *
 emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
 {
    struct ir3_block *b = ctx->block;
-   struct ir3_instruction *atomic;
-   type_t type = TYPE_U32;
+   nir_atomic_op op = nir_intrinsic_atomic_op(intr);
+   type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32;
 
    struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]);
 
@@ -161,50 +195,18 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    /* 64b byte offset */
    struct ir3_instruction *byte_offset =
       byte_offset_to_address(ctx, &intr->src[0], ir3_get_src(ctx, &intr->src[1])[0]);
-   /* dword offset for everything but comp_swap */
+   /* dword offset for everything but cmpxchg */
    struct ir3_instruction *src3 = ir3_get_src(ctx, &intr->src[3])[0];
 
-   switch (intr->intrinsic) {
-   case nir_intrinsic_ssbo_atomic_add_ir3:
-      atomic = ir3_ATOMIC_S_ADD(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_imin_ir3:
-      atomic = ir3_ATOMIC_S_MIN(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      type = TYPE_S32;
-      break;
-   case nir_intrinsic_ssbo_atomic_umin_ir3:
-      atomic = ir3_ATOMIC_S_MIN(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_imax_ir3:
-      atomic = ir3_ATOMIC_S_MAX(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      type = TYPE_S32;
-      break;
-   case nir_intrinsic_ssbo_atomic_umax_ir3:
-      atomic = ir3_ATOMIC_S_MAX(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_and_ir3:
-      atomic = ir3_ATOMIC_S_AND(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_or_ir3:
-      atomic = ir3_ATOMIC_S_OR(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_xor_ir3:
-      atomic = ir3_ATOMIC_S_XOR(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_exchange_ir3:
-      atomic = ir3_ATOMIC_S_XCHG(b, ssbo, 0, data, 0, src3, 0, byte_offset, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_comp_swap_ir3:
+   if (op == nir_atomic_op_cmpxchg) {
       /* for cmpxchg, src0 is [ui]vec2(data, compare): */
       data = ir3_collect(b, src3, data);
-      struct ir3_instruction *dword_offset = ir3_get_src(ctx, &intr->src[4])[0];
-      atomic = ir3_ATOMIC_S_CMPXCHG(b, ssbo, 0, data, 0, dword_offset, 0,
-                                    byte_offset, 0);
-      break;
-   default:
-      unreachable("boo");
+      src3 = ir3_get_src(ctx, &intr->src[4])[0];
    }
 
+   struct ir3_instruction *atomic =
+      emit_atomic(b, op, ssbo, data, src3, byte_offset);
+
    atomic->cat6.iim_val = 1;
    atomic->cat6.d = 4;
    atomic->cat6.type = type;
@@ -369,6 +371,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
    struct ir3_instruction *image = ir3_image_to_ibo(ctx, intr->src[0]);
    unsigned ncoords = ir3_get_image_coords(intr, NULL);
+   nir_atomic_op op = nir_intrinsic_atomic_op(intr);
 
    /* src0 is value (or uvec2(value, compare))
     * src1 is coords
@@ -378,39 +381,10 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    src1 = ir3_create_collect(b, coords, ncoords);
    src2 = get_image_offset(ctx, intr, coords, ctx->compiler->gen == 4);
 
-   switch (intr->intrinsic) {
-   case nir_intrinsic_image_atomic_add:
-      atomic = ir3_ATOMIC_S_ADD(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_image_atomic_umin:
-      atomic = ir3_ATOMIC_S_MIN(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_image_atomic_umax:
-      atomic = ir3_ATOMIC_S_MAX(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   case nir_intrinsic_image_atomic_and:
-      atomic = ir3_ATOMIC_S_AND(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   case nir_intrinsic_image_atomic_or:
-      atomic = ir3_ATOMIC_S_OR(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   case nir_intrinsic_image_atomic_xor:
-      atomic = ir3_ATOMIC_S_XOR(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   case nir_intrinsic_image_atomic_exchange:
-      atomic = ir3_ATOMIC_S_XCHG(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   case nir_intrinsic_image_atomic_comp_swap:
-      /* for cmpxchg, src0 is [ui]vec2(data, compare): */
+   if (op == nir_atomic_op_cmpxchg)
       src0 = ir3_collect(b, ir3_get_src(ctx, &intr->src[4])[0], src0);
-      atomic = ir3_ATOMIC_S_CMPXCHG(b, image, 0, src0, 0, src1, 0, src2, 0);
-      break;
-   default:
-      unreachable("boo");
-   }
 
+   atomic = emit_atomic(b, op, image, src0, src1, src2);
    atomic->cat6.iim_val = 1;
    atomic->cat6.d = ncoords;
    atomic->cat6.type = ir3_get_type_for_image_intrinsic(intr);
diff --git a/src/freedreno/ir3/ir3_a6xx.c b/src/freedreno/ir3/ir3_a6xx.c
index d5314c54ef5..c45f6108a32 100644
--- a/src/freedreno/ir3/ir3_a6xx.c
+++ b/src/freedreno/ir3/ir3_a6xx.c
@@ -89,6 +89,39 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    array_insert(b, b->keeps, stib);
 }
 
+static struct ir3_instruction *
+emit_atomic(struct ir3_block *b,
+            nir_atomic_op op,
+            struct ir3_instruction *ibo,
+            struct ir3_instruction *src0,
+            struct ir3_instruction *src1)
+{
+   switch (op) {
+   case nir_atomic_op_iadd:
+      return ir3_ATOMIC_B_ADD(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_imin:
+      return ir3_ATOMIC_B_MIN(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_umin:
+      return ir3_ATOMIC_B_MIN(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_imax:
+      return ir3_ATOMIC_B_MAX(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_umax:
+      return ir3_ATOMIC_B_MAX(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_iand:
+      return ir3_ATOMIC_B_AND(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_ior:
+      return ir3_ATOMIC_B_OR(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_ixor:
+      return ir3_ATOMIC_B_XOR(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_xchg:
+      return ir3_ATOMIC_B_XCHG(b, ibo, 0, src0, 0, src1, 0);
+   case nir_atomic_op_cmpxchg:
+      return ir3_ATOMIC_B_CMPXCHG(b, ibo, 0, src0, 0, src1, 0);
+   default:
+      unreachable("boo");
+   }
+}
+
 /*
  * SSBO atomic intrinsics
  *
@@ -103,7 +136,7 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
  * 1: The offset into the SSBO buffer of the variable that the atomic
  *    operation will operate on.
  * 2: The data parameter to the atomic function (i.e. the value to add
- *    in ssbo_atomic_add, etc).
+ *    in, etc).
  * 3: For CompSwap only: the second data parameter.
  */
 static struct ir3_instruction *
@@ -111,7 +144,8 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
 {
    struct ir3_block *b = ctx->block;
    struct ir3_instruction *atomic, *ibo, *src0, *src1, *data, *dummy;
-   type_t type = TYPE_U32;
+   nir_atomic_op op = nir_intrinsic_atomic_op(intr);
+   type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32;
 
    ibo = ir3_ssbo_to_ibo(ctx, intr->src[0]);
 
@@ -133,7 +167,7 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
     */
    dummy = create_immed(b, 0);
 
-   if (intr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap_ir3) {
+   if (op == nir_atomic_op_cmpxchg) {
       src0 = ir3_get_src(ctx, &intr->src[4])[0];
       struct ir3_instruction *compare = ir3_get_src(ctx, &intr->src[3])[0];
       src1 = ir3_collect(b, dummy, compare, data);
@@ -142,43 +176,7 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
       src1 = ir3_collect(b, dummy, data);
    }
 
-   switch (intr->intrinsic) {
-   case nir_intrinsic_ssbo_atomic_add_ir3:
-      atomic = ir3_ATOMIC_B_ADD(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_imin_ir3:
-      atomic = ir3_ATOMIC_B_MIN(b, ibo, 0, src0, 0, src1, 0);
-      type = TYPE_S32;
-      break;
-   case nir_intrinsic_ssbo_atomic_umin_ir3:
-      atomic = ir3_ATOMIC_B_MIN(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_imax_ir3:
-      atomic = ir3_ATOMIC_B_MAX(b, ibo, 0, src0, 0, src1, 0);
-      type = TYPE_S32;
-      break;
-   case nir_intrinsic_ssbo_atomic_umax_ir3:
-      atomic = ir3_ATOMIC_B_MAX(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_and_ir3:
-      atomic = ir3_ATOMIC_B_AND(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_or_ir3:
-      atomic = ir3_ATOMIC_B_OR(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_xor_ir3:
-      atomic = ir3_ATOMIC_B_XOR(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_exchange_ir3:
-      atomic = ir3_ATOMIC_B_XCHG(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_ssbo_atomic_comp_swap_ir3:
-      atomic = ir3_ATOMIC_B_CMPXCHG(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   default:
-      unreachable("boo");
-   }
-
+   atomic = emit_atomic(b, op, ibo, src0, src1);
    atomic->cat6.iim_val = 1;
    atomic->cat6.d = 1;
    atomic->cat6.type = type;
@@ -259,6 +257,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
    struct ir3_instruction *value = ir3_get_src(ctx, &intr->src[3])[0];
    unsigned ncoords = ir3_get_image_coords(intr, NULL);
+   nir_atomic_op op = nir_intrinsic_atomic_op(intr);
 
    ibo = ir3_image_to_ibo(ctx, intr->src[0]);
 
@@ -277,55 +276,14 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    dummy = create_immed(b, 0);
    src0 = ir3_create_collect(b, coords, ncoords);
 
-   if (intr->intrinsic == nir_intrinsic_image_atomic_comp_swap ||
-       intr->intrinsic == nir_intrinsic_bindless_image_atomic_comp_swap) {
+   if (op == nir_atomic_op_cmpxchg) {
       struct ir3_instruction *compare = ir3_get_src(ctx, &intr->src[4])[0];
       src1 = ir3_collect(b, dummy, compare, value);
    } else {
       src1 = ir3_collect(b, dummy, value);
    }
 
-   switch (intr->intrinsic) {
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_bindless_image_atomic_add:
-      atomic = ir3_ATOMIC_B_ADD(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_bindless_image_atomic_imin:
-   case nir_intrinsic_bindless_image_atomic_umin:
-      atomic = ir3_ATOMIC_B_MIN(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_bindless_image_atomic_imax:
-   case nir_intrinsic_bindless_image_atomic_umax:
-      atomic = ir3_ATOMIC_B_MAX(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_bindless_image_atomic_and:
-      atomic = ir3_ATOMIC_B_AND(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_bindless_image_atomic_or:
-      atomic = ir3_ATOMIC_B_OR(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_bindless_image_atomic_xor:
-      atomic = ir3_ATOMIC_B_XOR(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_bindless_image_atomic_exchange:
-      atomic = ir3_ATOMIC_B_XCHG(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_bindless_image_atomic_comp_swap:
-      atomic = ir3_ATOMIC_B_CMPXCHG(b, ibo, 0, src0, 0, src1, 0);
-      break;
-   default:
-      unreachable("boo");
-   }
-
+   atomic = emit_atomic(b, op, ibo, src0, src1);
    atomic->cat6.iim_val = 1;
    atomic->cat6.d = ncoords;
    atomic->cat6.type = ir3_get_type_for_image_intrinsic(intr);
@@ -447,49 +405,50 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    struct ir3_block *b = ctx->block;
    struct ir3_instruction *addr, *atomic, *src1;
    struct ir3_instruction *value = ir3_get_src(ctx, &intr->src[1])[0];
-   type_t type = TYPE_U32;
+   nir_atomic_op op = nir_intrinsic_atomic_op(intr);
+   type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32;
 
    addr = ir3_collect(b, ir3_get_src(ctx, &intr->src[0])[0],
                       ir3_get_src(ctx, &intr->src[0])[1]);
 
-   if (intr->intrinsic == nir_intrinsic_global_atomic_comp_swap_ir3) {
+   if (op == nir_atomic_op_cmpxchg) {
       struct ir3_instruction *compare = ir3_get_src(ctx, &intr->src[2])[0];
       src1 = ir3_collect(b, compare, value);
    } else {
       src1 = value;
    }
 
-   switch (intr->intrinsic) {
-   case nir_intrinsic_global_atomic_add_ir3:
+   switch (op) {
+   case nir_atomic_op_iadd:
       atomic = ir3_ATOMIC_G_ADD(b, addr, 0, src1, 0);
       break;
-   case nir_intrinsic_global_atomic_imin_ir3:
+   case nir_atomic_op_imin:
       atomic = ir3_ATOMIC_G_MIN(b, addr, 0, src1, 0);
       type = TYPE_S32;
       break;
-   case nir_intrinsic_global_atomic_umin_ir3:
+   case nir_atomic_op_umin:
       atomic = ir3_ATOMIC_G_MIN(b, addr, 0, src1, 0);
       break;
-   case nir_intrinsic_global_atomic_imax_ir3:
+   case nir_atomic_op_imax:
       atomic = ir3_ATOMIC_G_MAX(b, addr, 0, src1, 0);
       type = TYPE_S32;
       break;
-   case nir_intrinsic_global_atomic_umax_ir3:
+   case nir_atomic_op_umax:
       atomic = ir3_ATOMIC_G_MAX(b, addr, 0, src1, 0);
       break;
-   case nir_intrinsic_global_atomic_and_ir3:
+   case nir_atomic_op_iand:
       atomic = ir3_ATOMIC_G_AND(b, addr, 0, src1, 0);
       break;
-   case nir_intrinsic_global_atomic_or_ir3:
+   case nir_atomic_op_ior:
       atomic = ir3_ATOMIC_G_OR(b, addr, 0, src1, 0);
       break;
-   case nir_intrinsic_global_atomic_xor_ir3:
+   case nir_atomic_op_ixor:
       atomic = ir3_ATOMIC_G_XOR(b, addr, 0, src1, 0);
       break;
-   case nir_intrinsic_global_atomic_exchange_ir3:
+   case nir_atomic_op_xchg:
       atomic = ir3_ATOMIC_G_XCHG(b, addr, 0, src1, 0);
       break;
-   case nir_intrinsic_global_atomic_comp_swap_ir3:
+   case nir_atomic_op_cmpxchg:
       atomic = ir3_ATOMIC_G_CMPXCHG(b, addr, 0, src1, 0);
       break;
    default:
diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c
index 19f9cf6fd49..f8316f417a4 100644
--- a/src/freedreno/ir3/ir3_compiler_nir.c
+++ b/src/freedreno/ir3/ir3_compiler_nir.c
@@ -1192,7 +1192,7 @@ emit_intrinsic_store_shared_ir3(struct ir3_context *ctx,
  * 0: The offset into the shared variable storage region that the atomic
  *    operation will operate on.
  * 1: The data parameter to the atomic function (i.e. the value to add
- *    in shared_atomic_add, etc).
+ *    in, etc).
  * 2: For CompSwap only: the second data parameter.
  */
 static struct ir3_instruction *
@@ -1205,37 +1205,37 @@ emit_intrinsic_atomic_shared(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    src0 = ir3_get_src(ctx, &intr->src[0])[0]; /* offset */
    src1 = ir3_get_src(ctx, &intr->src[1])[0]; /* value */
 
-   switch (intr->intrinsic) {
-   case nir_intrinsic_shared_atomic_add:
+   switch (nir_intrinsic_atomic_op(intr)) {
+   case nir_atomic_op_iadd:
       atomic = ir3_ATOMIC_ADD(b, src0, 0, src1, 0);
       break;
-   case nir_intrinsic_shared_atomic_imin:
+   case nir_atomic_op_imin:
       atomic = ir3_ATOMIC_MIN(b, src0, 0, src1, 0);
       type = TYPE_S32;
       break;
-   case nir_intrinsic_shared_atomic_umin:
+   case nir_atomic_op_umin:
       atomic = ir3_ATOMIC_MIN(b, src0, 0, src1, 0);
       break;
-   case nir_intrinsic_shared_atomic_imax:
+   case nir_atomic_op_imax:
       atomic = ir3_ATOMIC_MAX(b, src0, 0, src1, 0);
       type = TYPE_S32;
       break;
-   case nir_intrinsic_shared_atomic_umax:
+   case nir_atomic_op_umax:
       atomic = ir3_ATOMIC_MAX(b, src0, 0, src1, 0);
       break;
-   case nir_intrinsic_shared_atomic_and:
+   case nir_atomic_op_iand:
       atomic = ir3_ATOMIC_AND(b, src0, 0, src1, 0);
       break;
-   case nir_intrinsic_shared_atomic_or:
+   case nir_atomic_op_ior:
       atomic = ir3_ATOMIC_OR(b, src0, 0, src1, 0);
       break;
-   case nir_intrinsic_shared_atomic_xor:
+   case nir_atomic_op_ixor:
       atomic = ir3_ATOMIC_XOR(b, src0, 0, src1, 0);
       break;
-   case nir_intrinsic_shared_atomic_exchange:
+   case nir_atomic_op_xchg:
       atomic = ir3_ATOMIC_XCHG(b, src0, 0, src1, 0);
       break;
-   case nir_intrinsic_shared_atomic_comp_swap:
+   case nir_atomic_op_cmpxchg:
       /* for cmpxchg, src1 is [ui]vec2(data, compare): */
       src1 = ir3_collect(b, ir3_get_src(ctx, &intr->src[2])[0], src1);
       atomic = ir3_ATOMIC_CMPXCHG(b, src0, 0, src1, 0);
@@ -2151,16 +2151,8 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    case nir_intrinsic_get_ssbo_size:
       emit_intrinsic_ssbo_size(ctx, intr, dst);
       break;
-   case nir_intrinsic_ssbo_atomic_add_ir3:
-   case nir_intrinsic_ssbo_atomic_imin_ir3:
-   case nir_intrinsic_ssbo_atomic_umin_ir3:
-   case nir_intrinsic_ssbo_atomic_imax_ir3:
-   case nir_intrinsic_ssbo_atomic_umax_ir3:
-   case nir_intrinsic_ssbo_atomic_and_ir3:
-   case nir_intrinsic_ssbo_atomic_or_ir3:
-   case nir_intrinsic_ssbo_atomic_xor_ir3:
-   case nir_intrinsic_ssbo_atomic_exchange_ir3:
-   case nir_intrinsic_ssbo_atomic_comp_swap_ir3:
+   case nir_intrinsic_ssbo_atomic_ir3:
+   case nir_intrinsic_ssbo_atomic_swap_ir3:
       dst[0] = ctx->funcs->emit_intrinsic_atomic_ssbo(ctx, intr);
       break;
    case nir_intrinsic_load_shared:
@@ -2169,16 +2161,8 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    case nir_intrinsic_store_shared:
       emit_intrinsic_store_shared(ctx, intr);
       break;
-   case nir_intrinsic_shared_atomic_add:
-   case nir_intrinsic_shared_atomic_imin:
-   case nir_intrinsic_shared_atomic_umin:
-   case nir_intrinsic_shared_atomic_imax:
-   case nir_intrinsic_shared_atomic_umax:
-   case nir_intrinsic_shared_atomic_and:
-   case nir_intrinsic_shared_atomic_or:
-   case nir_intrinsic_shared_atomic_xor:
-   case nir_intrinsic_shared_atomic_exchange:
-   case nir_intrinsic_shared_atomic_comp_swap:
+   case nir_intrinsic_shared_atomic:
+   case nir_intrinsic_shared_atomic_swap:
       dst[0] = emit_intrinsic_atomic_shared(ctx, intr);
       break;
    case nir_intrinsic_load_scratch:
@@ -2199,26 +2183,10 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    case nir_intrinsic_bindless_image_size:
       ctx->funcs->emit_intrinsic_image_size(ctx, intr, dst);
       break;
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_bindless_image_atomic_add:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_bindless_image_atomic_imin:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_bindless_image_atomic_umin:
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_bindless_image_atomic_imax:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_bindless_image_atomic_umax:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_bindless_image_atomic_and:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_bindless_image_atomic_or:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_bindless_image_atomic_xor:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_bindless_image_atomic_exchange:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_bindless_image_atomic_comp_swap:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
+   case nir_intrinsic_bindless_image_atomic_swap:
       dst[0] = ctx->funcs->emit_intrinsic_atomic_image(ctx, intr);
       break;
    case nir_intrinsic_scoped_barrier:
@@ -2591,16 +2559,8 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr)
    case nir_intrinsic_bindless_resource_ir3:
       dst[0] = ir3_get_src(ctx, &intr->src[0])[0];
       break;
-   case nir_intrinsic_global_atomic_add_ir3:
-   case nir_intrinsic_global_atomic_imin_ir3:
-   case nir_intrinsic_global_atomic_umin_ir3:
-   case nir_intrinsic_global_atomic_imax_ir3:
-   case nir_intrinsic_global_atomic_umax_ir3:
-   case nir_intrinsic_global_atomic_and_ir3:
-   case nir_intrinsic_global_atomic_or_ir3:
-   case nir_intrinsic_global_atomic_xor_ir3:
-   case nir_intrinsic_global_atomic_exchange_ir3:
-   case nir_intrinsic_global_atomic_comp_swap_ir3: {
+   case nir_intrinsic_global_atomic_ir3:
+   case nir_intrinsic_global_atomic_swap_ir3: {
       dst[0] = ctx->funcs->emit_intrinsic_atomic_global(ctx, intr);
       break;
    }
diff --git a/src/freedreno/ir3/ir3_image.c b/src/freedreno/ir3/ir3_image.c
index f52c188944a..9e732485f2f 100644
--- a/src/freedreno/ir3/ir3_image.c
+++ b/src/freedreno/ir3/ir3_image.c
@@ -136,32 +136,11 @@ ir3_get_type_for_image_intrinsic(const nir_intrinsic_instr *instr)
          type = nir_type_uint;
       break;
 
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_bindless_image_atomic_add:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_bindless_image_atomic_umin:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_bindless_image_atomic_umax:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_bindless_image_atomic_and:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_bindless_image_atomic_or:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_bindless_image_atomic_xor:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_bindless_image_atomic_exchange:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_bindless_image_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_inc_wrap:
-   case nir_intrinsic_bindless_image_atomic_inc_wrap:
-      type = nir_type_uint;
-      break;
-
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_bindless_image_atomic_imin:
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_bindless_image_atomic_imax:
-      type = nir_type_int;
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
+   case nir_intrinsic_bindless_image_atomic_swap:
+      type = nir_atomic_op_type(nir_intrinsic_atomic_op(instr));
       break;
 
    default:
diff --git a/src/freedreno/ir3/ir3_nir.c b/src/freedreno/ir3/ir3_nir.c
index 8bf3d720038..c0f40c7aa10 100644
--- a/src/freedreno/ir3/ir3_nir.c
+++ b/src/freedreno/ir3/ir3_nir.c
@@ -381,6 +381,9 @@ ir3_finalize_nir(struct ir3_compiler *compiler, nir_shader *s)
 
    OPT_V(s, nir_remove_dead_variables, nir_var_function_temp, NULL);
 
+   /* Temporary stopgap until the core is transitioned to unified atomics */
+   OPT_V(s, nir_lower_legacy_atomics);
+
    if (ir3_shader_debug & IR3_DBG_DISASM) {
       mesa_logi("----------------------");
       nir_log_shaderi(s);
@@ -842,16 +845,8 @@ ir3_nir_scan_driver_consts(struct ir3_compiler *compiler, nir_shader *shader, st
             unsigned idx;
 
             switch (intr->intrinsic) {
-            case nir_intrinsic_image_atomic_add:
-            case nir_intrinsic_image_atomic_imin:
-            case nir_intrinsic_image_atomic_umin:
-            case nir_intrinsic_image_atomic_imax:
-            case nir_intrinsic_image_atomic_umax:
-            case nir_intrinsic_image_atomic_and:
-            case nir_intrinsic_image_atomic_or:
-            case nir_intrinsic_image_atomic_xor:
-            case nir_intrinsic_image_atomic_exchange:
-            case nir_intrinsic_image_atomic_comp_swap:
+            case nir_intrinsic_image_atomic:
+            case nir_intrinsic_image_atomic_swap:
             case nir_intrinsic_image_load:
             case nir_intrinsic_image_store:
             case nir_intrinsic_image_size:
diff --git a/src/freedreno/ir3/ir3_nir_lower_64b.c b/src/freedreno/ir3/ir3_nir_lower_64b.c
index 9c6ae66dc09..f43205fe29c 100644
--- a/src/freedreno/ir3/ir3_nir_lower_64b.c
+++ b/src/freedreno/ir3/ir3_nir_lower_64b.c
@@ -233,16 +233,8 @@ lower_64b_global_filter(const nir_instr *instr, const void *unused)
    case nir_intrinsic_load_global:
    case nir_intrinsic_load_global_constant:
    case nir_intrinsic_store_global:
-   case nir_intrinsic_global_atomic_add:
-   case nir_intrinsic_global_atomic_imin:
-   case nir_intrinsic_global_atomic_umin:
-   case nir_intrinsic_global_atomic_imax:
-   case nir_intrinsic_global_atomic_umax:
-   case nir_intrinsic_global_atomic_and:
-   case nir_intrinsic_global_atomic_or:
-   case nir_intrinsic_global_atomic_xor:
-   case nir_intrinsic_global_atomic_exchange:
-   case nir_intrinsic_global_atomic_comp_swap:
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_global_atomic_swap:
       return true;
    default:
       return false;
@@ -265,31 +257,18 @@ lower_64b_global(nir_builder *b, nir_instr *instr, void *unused)
     * those up into max 4 components per load/store.
     */
 
-#define GLOBAL_IR3_2SRC(name)                                                 \
-   case nir_intrinsic_##name: {                                               \
-      return nir_build_##name##_ir3(b, nir_dest_bit_size(intr->dest), addr,   \
-                                  nir_ssa_for_src(b, intr->src[1], 1));       \
-   }
-
-   switch (intr->intrinsic) {
-   GLOBAL_IR3_2SRC(global_atomic_add)
-   GLOBAL_IR3_2SRC(global_atomic_imin)
-   GLOBAL_IR3_2SRC(global_atomic_umin)
-   GLOBAL_IR3_2SRC(global_atomic_imax)
-   GLOBAL_IR3_2SRC(global_atomic_umax)
-   GLOBAL_IR3_2SRC(global_atomic_and)
-   GLOBAL_IR3_2SRC(global_atomic_or)
-   GLOBAL_IR3_2SRC(global_atomic_xor)
-   GLOBAL_IR3_2SRC(global_atomic_exchange)
-   case nir_intrinsic_global_atomic_comp_swap:
-      return nir_build_global_atomic_comp_swap_ir3(
+   if (intr->intrinsic == nir_intrinsic_global_atomic) {
+      return nir_build_global_atomic_ir3(
+            b, nir_dest_bit_size(intr->dest), addr,
+            nir_ssa_for_src(b, intr->src[1], 1),
+         .atomic_op = nir_intrinsic_atomic_op(intr));
+   } else if (intr->intrinsic == nir_intrinsic_global_atomic_swap) {
+      return nir_build_global_atomic_swap_ir3(
          b, nir_dest_bit_size(intr->dest), addr,
          nir_ssa_for_src(b, intr->src[1], 1),
-         nir_ssa_for_src(b, intr->src[2], 1));
-   default:
-      break;
+         nir_ssa_for_src(b, intr->src[2], 1),
+         .atomic_op = nir_intrinsic_atomic_op(intr));
    }
-#undef GLOBAL_IR3_2SRC
 
    if (load) {
       unsigned num_comp = nir_intrinsic_dest_components(intr);
diff --git a/src/freedreno/ir3/ir3_nir_lower_io_offsets.c b/src/freedreno/ir3/ir3_nir_lower_io_offsets.c
index d7df5b8ecfc..280ab756296 100644
--- a/src/freedreno/ir3/ir3_nir_lower_io_offsets.c
+++ b/src/freedreno/ir3/ir3_nir_lower_io_offsets.c
@@ -55,26 +55,10 @@ get_ir3_intrinsic_for_ssbo_intrinsic(unsigned intrinsic,
       return nir_intrinsic_store_ssbo_ir3;
    case nir_intrinsic_load_ssbo:
       return nir_intrinsic_load_ssbo_ir3;
-   case nir_intrinsic_ssbo_atomic_add:
-      return nir_intrinsic_ssbo_atomic_add_ir3;
-   case nir_intrinsic_ssbo_atomic_imin:
-      return nir_intrinsic_ssbo_atomic_imin_ir3;
-   case nir_intrinsic_ssbo_atomic_umin:
-      return nir_intrinsic_ssbo_atomic_umin_ir3;
-   case nir_intrinsic_ssbo_atomic_imax:
-      return nir_intrinsic_ssbo_atomic_imax_ir3;
-   case nir_intrinsic_ssbo_atomic_umax:
-      return nir_intrinsic_ssbo_atomic_umax_ir3;
-   case nir_intrinsic_ssbo_atomic_and:
-      return nir_intrinsic_ssbo_atomic_and_ir3;
-   case nir_intrinsic_ssbo_atomic_or:
-      return nir_intrinsic_ssbo_atomic_or_ir3;
-   case nir_intrinsic_ssbo_atomic_xor:
-      return nir_intrinsic_ssbo_atomic_xor_ir3;
-   case nir_intrinsic_ssbo_atomic_exchange:
-      return nir_intrinsic_ssbo_atomic_exchange_ir3;
-   case nir_intrinsic_ssbo_atomic_comp_swap:
-      return nir_intrinsic_ssbo_atomic_comp_swap_ir3;
+   case nir_intrinsic_ssbo_atomic:
+      return nir_intrinsic_ssbo_atomic_ir3;
+   case nir_intrinsic_ssbo_atomic_swap:
+      return nir_intrinsic_ssbo_atomic_swap_ir3;
    default:
       break;
    }