From b838a8746bff6ee346a5bbf4469604383185bc5b Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 12 May 2023 09:14:30 -0400 Subject: glsl/nir: Produce unified atomics Signed-off-by: Alyssa Rosenzweig Reviewed-by: Emma Anholt Reviewed-by: Jesse Natalie Part-of: --- src/compiler/glsl/gl_nir_link_uniforms.c | 12 +- src/compiler/glsl/gl_nir_lower_images.c | 15 +- src/compiler/glsl/gl_nir_lower_samplers_as_deref.c | 13 +- src/compiler/glsl/glsl_to_nir.cpp | 179 ++++++++++----------- 4 files changed, 95 insertions(+), 124 deletions(-) diff --git a/src/compiler/glsl/gl_nir_link_uniforms.c b/src/compiler/glsl/gl_nir_link_uniforms.c index f9721393fbc..83054469d3a 100644 --- a/src/compiler/glsl/gl_nir_link_uniforms.c +++ b/src/compiler/glsl/gl_nir_link_uniforms.c @@ -546,16 +546,8 @@ add_var_use_shader(nir_shader *shader, struct hash_table *live) case nir_intrinsic_atomic_counter_comp_swap_deref: case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: - case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_umin: - case nir_intrinsic_image_deref_atomic_imin: - case nir_intrinsic_image_deref_atomic_umax: - case nir_intrinsic_image_deref_atomic_imax: - case nir_intrinsic_image_deref_atomic_and: - case nir_intrinsic_image_deref_atomic_or: - case nir_intrinsic_image_deref_atomic_xor: - case nir_intrinsic_image_deref_atomic_exchange: - case nir_intrinsic_image_deref_atomic_comp_swap: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_size: case nir_intrinsic_image_deref_samples: case nir_intrinsic_load_deref: diff --git a/src/compiler/glsl/gl_nir_lower_images.c b/src/compiler/glsl/gl_nir_lower_images.c index 9bf2129f4b7..4d2dd2b4ac3 100644 --- a/src/compiler/glsl/gl_nir_lower_images.c +++ b/src/compiler/glsl/gl_nir_lower_images.c @@ -64,19 +64,8 @@ lower_instr(nir_builder *b, nir_instr *instr, void *cb_data) nir_variable *var; switch (intrinsic->intrinsic) { - case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_imin: - case nir_intrinsic_image_deref_atomic_umin: - case nir_intrinsic_image_deref_atomic_imax: - case nir_intrinsic_image_deref_atomic_umax: - case nir_intrinsic_image_deref_atomic_and: - case nir_intrinsic_image_deref_atomic_or: - case nir_intrinsic_image_deref_atomic_xor: - case nir_intrinsic_image_deref_atomic_exchange: - case nir_intrinsic_image_deref_atomic_comp_swap: - case nir_intrinsic_image_deref_atomic_fadd: - case nir_intrinsic_image_deref_atomic_inc_wrap: - case nir_intrinsic_image_deref_atomic_dec_wrap: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_samples: case nir_intrinsic_image_deref_size: diff --git a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c index 934369d74fe..7b529bc4347 100644 --- a/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c +++ b/src/compiler/glsl/gl_nir_lower_samplers_as_deref.c @@ -316,17 +316,8 @@ lower_intrinsic(nir_intrinsic_instr *instr, { if (instr->intrinsic == nir_intrinsic_image_deref_load || instr->intrinsic == nir_intrinsic_image_deref_store || - instr->intrinsic == nir_intrinsic_image_deref_atomic_add || - instr->intrinsic == nir_intrinsic_image_deref_atomic_imin || - instr->intrinsic == nir_intrinsic_image_deref_atomic_umin || - instr->intrinsic == nir_intrinsic_image_deref_atomic_imax || - instr->intrinsic == nir_intrinsic_image_deref_atomic_umax || - instr->intrinsic == nir_intrinsic_image_deref_atomic_and || - instr->intrinsic == nir_intrinsic_image_deref_atomic_or || - instr->intrinsic == nir_intrinsic_image_deref_atomic_xor || - instr->intrinsic == nir_intrinsic_image_deref_atomic_exchange || - instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap || - instr->intrinsic == nir_intrinsic_image_deref_atomic_fadd || + instr->intrinsic == nir_intrinsic_image_deref_atomic || + instr->intrinsic == nir_intrinsic_image_deref_atomic_swap || instr->intrinsic == nir_intrinsic_image_deref_size || instr->intrinsic == nir_intrinsic_image_deref_samples_identical || instr->intrinsic == nir_intrinsic_image_deref_descriptor_amd || diff --git a/src/compiler/glsl/glsl_to_nir.cpp b/src/compiler/glsl/glsl_to_nir.cpp index 00b6afa1a20..3b2ec155bf6 100644 --- a/src/compiler/glsl/glsl_to_nir.cpp +++ b/src/compiler/glsl/glsl_to_nir.cpp @@ -970,53 +970,64 @@ nir_visitor::visit(ir_call *ir) if (ir->callee->is_intrinsic()) { nir_intrinsic_op op; + /* Initialize to something because gcc complains otherwise */ + nir_atomic_op atomic_op = nir_atomic_op_iadd; + switch (ir->callee->intrinsic_id) { case ir_intrinsic_generic_atomic_add: - op = ir->return_deref->type->is_integer_32_64() - ? nir_intrinsic_deref_atomic_add : nir_intrinsic_deref_atomic_fadd; + op = nir_intrinsic_deref_atomic; + atomic_op = ir->return_deref->type->is_integer_32_64() + ? nir_atomic_op_iadd : nir_atomic_op_fadd; break; case ir_intrinsic_generic_atomic_and: - op = nir_intrinsic_deref_atomic_and; + op = nir_intrinsic_deref_atomic; + atomic_op = nir_atomic_op_iand; break; case ir_intrinsic_generic_atomic_or: - op = nir_intrinsic_deref_atomic_or; + op = nir_intrinsic_deref_atomic; + atomic_op = nir_atomic_op_ior; break; case ir_intrinsic_generic_atomic_xor: - op = nir_intrinsic_deref_atomic_xor; + op = nir_intrinsic_deref_atomic; + atomic_op = nir_atomic_op_ixor; break; case ir_intrinsic_generic_atomic_min: assert(ir->return_deref); + op = nir_intrinsic_deref_atomic; if (ir->return_deref->type == glsl_type::int_type || ir->return_deref->type == glsl_type::int64_t_type) - op = nir_intrinsic_deref_atomic_imin; + atomic_op = nir_atomic_op_imin; else if (ir->return_deref->type == glsl_type::uint_type || ir->return_deref->type == glsl_type::uint64_t_type) - op = nir_intrinsic_deref_atomic_umin; + atomic_op = nir_atomic_op_umin; else if (ir->return_deref->type == glsl_type::float_type) - op = nir_intrinsic_deref_atomic_fmin; + atomic_op = nir_atomic_op_fmin; else unreachable("Invalid type"); break; case ir_intrinsic_generic_atomic_max: assert(ir->return_deref); + op = nir_intrinsic_deref_atomic; if (ir->return_deref->type == glsl_type::int_type || ir->return_deref->type == glsl_type::int64_t_type) - op = nir_intrinsic_deref_atomic_imax; + atomic_op = nir_atomic_op_imax; else if (ir->return_deref->type == glsl_type::uint_type || ir->return_deref->type == glsl_type::uint64_t_type) - op = nir_intrinsic_deref_atomic_umax; + atomic_op = nir_atomic_op_umax; else if (ir->return_deref->type == glsl_type::float_type) - op = nir_intrinsic_deref_atomic_fmax; + atomic_op = nir_atomic_op_fmax; else unreachable("Invalid type"); break; case ir_intrinsic_generic_atomic_exchange: - op = nir_intrinsic_deref_atomic_exchange; + op = nir_intrinsic_deref_atomic; + atomic_op = nir_atomic_op_xchg; break; case ir_intrinsic_generic_atomic_comp_swap: - op = ir->return_deref->type->is_integer_32_64() - ? nir_intrinsic_deref_atomic_comp_swap - : nir_intrinsic_deref_atomic_fcomp_swap; + op = nir_intrinsic_deref_atomic_swap; + atomic_op = ir->return_deref->type->is_integer_32_64() + ? nir_atomic_op_cmpxchg + : nir_atomic_op_fcmpxchg; break; case ir_intrinsic_atomic_counter_read: op = nir_intrinsic_atomic_counter_read_deref; @@ -1058,46 +1069,56 @@ nir_visitor::visit(ir_call *ir) op = nir_intrinsic_image_deref_store; break; case ir_intrinsic_image_atomic_add: - op = ir->return_deref->type->is_integer_32_64() - ? nir_intrinsic_image_deref_atomic_add - : nir_intrinsic_image_deref_atomic_fadd; + op = nir_intrinsic_image_deref_atomic; + atomic_op = ir->return_deref->type->is_integer_32_64() + ? nir_atomic_op_iadd + : nir_atomic_op_fadd; break; case ir_intrinsic_image_atomic_min: + op = nir_intrinsic_image_deref_atomic; if (ir->return_deref->type == glsl_type::int_type) - op = nir_intrinsic_image_deref_atomic_imin; + atomic_op = nir_atomic_op_imin; else if (ir->return_deref->type == glsl_type::uint_type) - op = nir_intrinsic_image_deref_atomic_umin; + atomic_op = nir_atomic_op_umin; else unreachable("Invalid type"); break; case ir_intrinsic_image_atomic_max: + op = nir_intrinsic_image_deref_atomic; if (ir->return_deref->type == glsl_type::int_type) - op = nir_intrinsic_image_deref_atomic_imax; + atomic_op = nir_atomic_op_imax; else if (ir->return_deref->type == glsl_type::uint_type) - op = nir_intrinsic_image_deref_atomic_umax; + atomic_op = nir_atomic_op_umax; else unreachable("Invalid type"); break; case ir_intrinsic_image_atomic_and: - op = nir_intrinsic_image_deref_atomic_and; + op = nir_intrinsic_image_deref_atomic; + atomic_op = nir_atomic_op_iand; break; case ir_intrinsic_image_atomic_or: - op = nir_intrinsic_image_deref_atomic_or; + op = nir_intrinsic_image_deref_atomic; + atomic_op = nir_atomic_op_ior; break; case ir_intrinsic_image_atomic_xor: - op = nir_intrinsic_image_deref_atomic_xor; + op = nir_intrinsic_image_deref_atomic; + atomic_op = nir_atomic_op_ixor; break; case ir_intrinsic_image_atomic_exchange: - op = nir_intrinsic_image_deref_atomic_exchange; + op = nir_intrinsic_image_deref_atomic; + atomic_op = nir_atomic_op_xchg; break; case ir_intrinsic_image_atomic_comp_swap: - op = nir_intrinsic_image_deref_atomic_comp_swap; + op = nir_intrinsic_image_deref_atomic_swap; + atomic_op = nir_atomic_op_cmpxchg; break; case ir_intrinsic_image_atomic_inc_wrap: - op = nir_intrinsic_image_deref_atomic_inc_wrap; + op = nir_intrinsic_image_deref_atomic; + atomic_op = nir_atomic_op_inc_wrap; break; case ir_intrinsic_image_atomic_dec_wrap: - op = nir_intrinsic_image_deref_atomic_dec_wrap; + op = nir_intrinsic_image_deref_atomic; + atomic_op = nir_atomic_op_dec_wrap; break; case ir_intrinsic_memory_barrier: op = shader->options->use_scoped_barrier @@ -1154,52 +1175,60 @@ nir_visitor::visit(ir_call *ir) op = nir_intrinsic_store_shared; break; case ir_intrinsic_shared_atomic_add: - op = ir->return_deref->type->is_integer_32_64() - ? nir_intrinsic_shared_atomic_add - : nir_intrinsic_shared_atomic_fadd; + op = nir_intrinsic_shared_atomic; + atomic_op = ir->return_deref->type->is_integer_32_64() + ? nir_atomic_op_iadd + : nir_atomic_op_fadd; break; case ir_intrinsic_shared_atomic_and: - op = nir_intrinsic_shared_atomic_and; + op = nir_intrinsic_shared_atomic; + atomic_op = nir_atomic_op_iand; break; case ir_intrinsic_shared_atomic_or: - op = nir_intrinsic_shared_atomic_or; + op = nir_intrinsic_shared_atomic; + atomic_op = nir_atomic_op_ior; break; case ir_intrinsic_shared_atomic_xor: - op = nir_intrinsic_shared_atomic_xor; + op = nir_intrinsic_shared_atomic; + atomic_op = nir_atomic_op_ixor; break; case ir_intrinsic_shared_atomic_min: assert(ir->return_deref); + op = nir_intrinsic_shared_atomic; if (ir->return_deref->type == glsl_type::int_type || ir->return_deref->type == glsl_type::int64_t_type) - op = nir_intrinsic_shared_atomic_imin; + atomic_op = nir_atomic_op_imin; else if (ir->return_deref->type == glsl_type::uint_type || ir->return_deref->type == glsl_type::uint64_t_type) - op = nir_intrinsic_shared_atomic_umin; + atomic_op = nir_atomic_op_umin; else if (ir->return_deref->type == glsl_type::float_type) - op = nir_intrinsic_shared_atomic_fmin; + atomic_op = nir_atomic_op_fmin; else unreachable("Invalid type"); break; case ir_intrinsic_shared_atomic_max: assert(ir->return_deref); + op = nir_intrinsic_shared_atomic; if (ir->return_deref->type == glsl_type::int_type || ir->return_deref->type == glsl_type::int64_t_type) - op = nir_intrinsic_shared_atomic_imax; + atomic_op = nir_atomic_op_imax; else if (ir->return_deref->type == glsl_type::uint_type || ir->return_deref->type == glsl_type::uint64_t_type) - op = nir_intrinsic_shared_atomic_umax; + atomic_op = nir_atomic_op_umax; else if (ir->return_deref->type == glsl_type::float_type) - op = nir_intrinsic_shared_atomic_fmax; + atomic_op = nir_atomic_op_fmax; else unreachable("Invalid type"); break; case ir_intrinsic_shared_atomic_exchange: - op = nir_intrinsic_shared_atomic_exchange; + op = nir_intrinsic_shared_atomic; + atomic_op = nir_atomic_op_xchg; break; case ir_intrinsic_shared_atomic_comp_swap: - op = ir->return_deref->type->is_integer_32_64() - ? nir_intrinsic_shared_atomic_comp_swap - : nir_intrinsic_shared_atomic_fcomp_swap; + op = nir_intrinsic_shared_atomic_swap; + atomic_op = ir->return_deref->type->is_integer_32_64() + ? nir_atomic_op_cmpxchg + : nir_atomic_op_fcmpxchg; break; case ir_intrinsic_vote_any: op = nir_intrinsic_vote_any; @@ -1233,20 +1262,8 @@ nir_visitor::visit(ir_call *ir) nir_ssa_def *ret = &instr->dest.ssa; switch (op) { - case nir_intrinsic_deref_atomic_add: - case nir_intrinsic_deref_atomic_imin: - case nir_intrinsic_deref_atomic_umin: - case nir_intrinsic_deref_atomic_imax: - case nir_intrinsic_deref_atomic_umax: - case nir_intrinsic_deref_atomic_and: - case nir_intrinsic_deref_atomic_or: - case nir_intrinsic_deref_atomic_xor: - case nir_intrinsic_deref_atomic_exchange: - case nir_intrinsic_deref_atomic_comp_swap: - case nir_intrinsic_deref_atomic_fadd: - case nir_intrinsic_deref_atomic_fmin: - case nir_intrinsic_deref_atomic_fmax: - case nir_intrinsic_deref_atomic_fcomp_swap: { + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: { int param_count = ir->actual_parameters.length(); assert(param_count == 2 || param_count == 3); @@ -1269,6 +1286,7 @@ nir_visitor::visit(ir_call *ir) } instr->src[0] = nir_src_for_ssa(&nir_deref->dest.ssa); + nir_intrinsic_set_atomic_op(instr, atomic_op); nir_intrinsic_set_access(instr, deref_get_qualifier(nir_deref)); /* data1 parameter (this is always present) */ @@ -1278,8 +1296,7 @@ nir_visitor::visit(ir_call *ir) /* data2 parameter (only with atomic_comp_swap) */ if (param_count == 3) { - assert(op == nir_intrinsic_deref_atomic_comp_swap || - op == nir_intrinsic_deref_atomic_fcomp_swap); + assert(op == nir_intrinsic_deref_atomic_swap); param = param->get_next(); inst = (ir_instruction *) param; instr->src[2] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue())); @@ -1338,21 +1355,10 @@ nir_visitor::visit(ir_call *ir) } case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: - case nir_intrinsic_image_deref_atomic_add: - case nir_intrinsic_image_deref_atomic_imin: - case nir_intrinsic_image_deref_atomic_umin: - case nir_intrinsic_image_deref_atomic_imax: - case nir_intrinsic_image_deref_atomic_umax: - case nir_intrinsic_image_deref_atomic_and: - case nir_intrinsic_image_deref_atomic_or: - case nir_intrinsic_image_deref_atomic_xor: - case nir_intrinsic_image_deref_atomic_exchange: - case nir_intrinsic_image_deref_atomic_comp_swap: - case nir_intrinsic_image_deref_atomic_fadd: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_samples: case nir_intrinsic_image_deref_size: - case nir_intrinsic_image_deref_atomic_inc_wrap: - case nir_intrinsic_image_deref_atomic_dec_wrap: case nir_intrinsic_image_deref_sparse_load: { /* Set the image variable dereference. */ exec_node *param = ir->actual_parameters.get_head(); @@ -1362,6 +1368,11 @@ nir_visitor::visit(ir_call *ir) nir_intrinsic_set_access(instr, deref_get_qualifier(deref)); + if (op == nir_intrinsic_image_deref_atomic || + op == nir_intrinsic_image_deref_atomic_swap) { + nir_intrinsic_set_atomic_op(instr, atomic_op); + } + instr->src[0] = nir_src_for_ssa(&deref->dest.ssa); param = param->get_next(); nir_intrinsic_set_image_dim(instr, @@ -1610,20 +1621,8 @@ nir_visitor::visit(ir_call *ir) nir_builder_instr_insert(&b, &instr->instr); break; } - case nir_intrinsic_shared_atomic_add: - case nir_intrinsic_shared_atomic_imin: - case nir_intrinsic_shared_atomic_umin: - case nir_intrinsic_shared_atomic_imax: - case nir_intrinsic_shared_atomic_umax: - case nir_intrinsic_shared_atomic_and: - case nir_intrinsic_shared_atomic_or: - case nir_intrinsic_shared_atomic_xor: - case nir_intrinsic_shared_atomic_exchange: - case nir_intrinsic_shared_atomic_comp_swap: - case nir_intrinsic_shared_atomic_fadd: - case nir_intrinsic_shared_atomic_fmin: - case nir_intrinsic_shared_atomic_fmax: - case nir_intrinsic_shared_atomic_fcomp_swap: { + case nir_intrinsic_shared_atomic: + case nir_intrinsic_shared_atomic_swap: { int param_count = ir->actual_parameters.length(); assert(param_count == 2 || param_count == 3); @@ -1639,8 +1638,7 @@ nir_visitor::visit(ir_call *ir) /* data2 parameter (only with atomic_comp_swap) */ if (param_count == 3) { - assert(op == nir_intrinsic_shared_atomic_comp_swap || - op == nir_intrinsic_shared_atomic_fcomp_swap); + assert(op == nir_intrinsic_shared_atomic_swap); param = param->get_next(); inst = (ir_instruction *) param; instr->src[2] = @@ -1653,6 +1651,7 @@ nir_visitor::visit(ir_call *ir) nir_ssa_dest_init(&instr->instr, &instr->dest, ir->return_deref->type->vector_elements, bit_size, NULL); + nir_intrinsic_set_atomic_op(instr, atomic_op); nir_builder_instr_insert(&b, &instr->instr); break; } -- cgit v1.2.1