radv: move queue related code to radv_queue.c

radv_device.c is getting too big and this follows the Vulkan common
runtime infrastructure.

Signed-off-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/21767>

1 files changed, 0 insertions, 1858 deletions

diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c
index e988e6a1722..d1342475eb0 100644
--- a/src/amd/vulkan/radv_device.c
+++ b/src/amd/vulkan/radv_device.c
@@ -78,15 +78,6 @@ typedef void *drmDevicePtr;
 #include "ac_llvm_util.h"
 #endif
 
-/* The number of IBs per submit isn't infinite, it depends on the IP type
- * (ie. some initial setup needed for a submit) and the number of IBs (4 DW).
- * This limit is arbitrary but should be safe for now.  Ideally, we should get
- * this limit from the KMD.
- */
-#define RADV_MAX_IBS_PER_SUBMIT 192
-
-static VkResult radv_queue_submit(struct vk_queue *vqueue, struct vk_queue_submit *submission);
-
 int
 radv_get_int_debug_option(const char *name, int default_value)
 {
@@ -142,152 +133,6 @@ radv_GetMemoryHostPointerPropertiesEXT(
    }
 }
 
-static enum radeon_ctx_priority
-radv_get_queue_global_priority(const VkDeviceQueueGlobalPriorityCreateInfoKHR *pObj)
-{
-   /* Default to MEDIUM when a specific global priority isn't requested */
-   if (!pObj)
-      return RADEON_CTX_PRIORITY_MEDIUM;
-
-   switch (pObj->globalPriority) {
-   case VK_QUEUE_GLOBAL_PRIORITY_REALTIME_KHR:
-      return RADEON_CTX_PRIORITY_REALTIME;
-   case VK_QUEUE_GLOBAL_PRIORITY_HIGH_KHR:
-      return RADEON_CTX_PRIORITY_HIGH;
-   case VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_KHR:
-      return RADEON_CTX_PRIORITY_MEDIUM;
-   case VK_QUEUE_GLOBAL_PRIORITY_LOW_KHR:
-      return RADEON_CTX_PRIORITY_LOW;
-   default:
-      unreachable("Illegal global priority value");
-      return RADEON_CTX_PRIORITY_INVALID;
-   }
-}
-
-int
-radv_queue_init(struct radv_device *device, struct radv_queue *queue, int idx,
-                const VkDeviceQueueCreateInfo *create_info,
-                const VkDeviceQueueGlobalPriorityCreateInfoKHR *global_priority)
-{
-   queue->device = device;
-   queue->priority = radv_get_queue_global_priority(global_priority);
-   queue->hw_ctx = device->hw_ctx[queue->priority];
-   queue->state.qf = vk_queue_to_radv(device->physical_device, create_info->queueFamilyIndex);
-   queue->gang_sem_bo = NULL;
-
-   VkResult result = vk_queue_init(&queue->vk, &device->vk, create_info, idx);
-   if (result != VK_SUCCESS)
-      return result;
-
-   queue->state.uses_shadow_regs =
-      device->uses_shadow_regs && queue->state.qf == RADV_QUEUE_GENERAL;
-   if (queue->state.uses_shadow_regs) {
-      result = radv_create_shadow_regs_preamble(device, &queue->state);
-      if (result != VK_SUCCESS)
-         goto fail;
-      result = radv_init_shadowed_regs_buffer_state(device, queue);
-      if (result != VK_SUCCESS)
-         goto fail;
-   }
-
-   queue->vk.driver_submit = radv_queue_submit;
-   return VK_SUCCESS;
-fail:
-   vk_queue_finish(&queue->vk);
-   return result;
-}
-
-static void
-radv_queue_state_finish(struct radv_queue_state *queue, struct radv_device *device)
-{
-   radv_destroy_shadow_regs_preamble(queue, device->ws);
-   if (queue->initial_full_flush_preamble_cs)
-      device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
-   if (queue->initial_preamble_cs)
-      device->ws->cs_destroy(queue->initial_preamble_cs);
-   if (queue->continue_preamble_cs)
-      device->ws->cs_destroy(queue->continue_preamble_cs);
-   if (queue->gang_wait_preamble_cs)
-      device->ws->cs_destroy(queue->gang_wait_preamble_cs);
-   if (queue->gang_wait_postamble_cs)
-      device->ws->cs_destroy(queue->gang_wait_postamble_cs);
-   if (queue->descriptor_bo)
-      device->ws->buffer_destroy(device->ws, queue->descriptor_bo);
-   if (queue->scratch_bo) {
-      device->ws->buffer_destroy(device->ws, queue->scratch_bo);
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->scratch_bo);
-   }
-   if (queue->esgs_ring_bo) {
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->esgs_ring_bo);
-      device->ws->buffer_destroy(device->ws, queue->esgs_ring_bo);
-   }
-   if (queue->gsvs_ring_bo) {
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->gsvs_ring_bo);
-      device->ws->buffer_destroy(device->ws, queue->gsvs_ring_bo);
-   }
-   if (queue->tess_rings_bo) {
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->tess_rings_bo);
-      device->ws->buffer_destroy(device->ws, queue->tess_rings_bo);
-   }
-   if (queue->task_rings_bo) {
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->task_rings_bo);
-      device->ws->buffer_destroy(device->ws, queue->task_rings_bo);
-   }
-   if (queue->mesh_scratch_ring_bo) {
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->mesh_scratch_ring_bo);
-      device->ws->buffer_destroy(device->ws, queue->mesh_scratch_ring_bo);
-   }
-   if (queue->attr_ring_bo) {
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->attr_ring_bo);
-      device->ws->buffer_destroy(device->ws, queue->attr_ring_bo);
-   }
-   if (queue->gds_bo) {
-      device->ws->buffer_destroy(device->ws, queue->gds_bo);
-      device->ws->buffer_make_resident(device->ws, queue->gds_bo, false);
-   }
-   if (queue->gds_oa_bo) {
-      device->ws->buffer_destroy(device->ws, queue->gds_oa_bo);
-      device->ws->buffer_make_resident(device->ws, queue->gds_oa_bo, false);
-   }
-   if (queue->compute_scratch_bo) {
-      radv_rmv_log_command_buffer_bo_destroy(device, queue->compute_scratch_bo);
-      device->ws->buffer_destroy(device->ws, queue->compute_scratch_bo);
-   }
-}
-
-static void
-radv_queue_finish(struct radv_queue *queue)
-{
-   if (queue->ace_internal_state) {
-      /* Prevent double free */
-      queue->ace_internal_state->task_rings_bo = NULL;
-
-      /* Clean up the internal ACE queue state. */
-      radv_queue_state_finish(queue->ace_internal_state, queue->device);
-      free(queue->ace_internal_state);
-   }
-
-   if (queue->gang_sem_bo)
-      queue->device->ws->buffer_destroy(queue->device->ws, queue->gang_sem_bo);
-
-   radv_queue_state_finish(&queue->state, queue->device);
-   vk_queue_finish(&queue->vk);
-}
-
-static bool
-radv_queue_init_ace_internal_state(struct radv_queue *queue)
-{
-   if (queue->ace_internal_state)
-      return true;
-
-   queue->ace_internal_state = calloc(1, sizeof(struct radv_queue_state));
-   if (!queue->ace_internal_state)
-      return false;
-
-   queue->ace_internal_state->qf = RADV_QUEUE_COMPUTE;
-   return true;
-}
-
 static VkResult
 radv_device_init_border_color(struct radv_device *device)
 {
@@ -1326,1709 +1171,6 @@ radv_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator)
    vk_free(&device->vk.alloc, device);
 }
 
-static void
-radv_fill_shader_rings(struct radv_device *device, uint32_t *map, bool add_sample_positions,
-                       uint32_t esgs_ring_size, struct radeon_winsys_bo *esgs_ring_bo,
-                       uint32_t gsvs_ring_size, struct radeon_winsys_bo *gsvs_ring_bo,
-                       struct radeon_winsys_bo *tess_rings_bo,
-                       struct radeon_winsys_bo *task_rings_bo,
-                       struct radeon_winsys_bo *mesh_scratch_ring_bo,
-                       uint32_t attr_ring_size, struct radeon_winsys_bo *attr_ring_bo)
-{
-   uint32_t *desc = &map[4];
-
-   if (esgs_ring_bo) {
-      uint64_t esgs_va = radv_buffer_get_va(esgs_ring_bo);
-
-      /* stride 0, num records - size, add tid, swizzle, elsize4,
-         index stride 64 */
-      desc[0] = esgs_va;
-      desc[1] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32);
-      desc[2] = esgs_ring_size;
-      desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
-                S_008F0C_INDEX_STRIDE(3) | S_008F0C_ADD_TID_ENABLE(1);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11)
-         desc[1] |= S_008F04_SWIZZLE_ENABLE_GFX11(1);
-      else
-         desc[1] |= S_008F04_SWIZZLE_ENABLE_GFX6(1);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX8) {
-         /* DATA_FORMAT is STRIDE[14:17] for MUBUF with ADD_TID_ENABLE=1 */
-         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(0) | S_008F0C_ELEMENT_SIZE(1);
-      } else {
-         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) | S_008F0C_ELEMENT_SIZE(1);
-      }
-
-      /* GS entry for ES->GS ring */
-      /* stride 0, num records - size, elsize0,
-         index stride 0 */
-      desc[4] = esgs_va;
-      desc[5] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32);
-      desc[6] = esgs_ring_size;
-      desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1);
-      } else {
-         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
-      }
-   }
-
-   desc += 8;
-
-   if (gsvs_ring_bo) {
-      uint64_t gsvs_va = radv_buffer_get_va(gsvs_ring_bo);
-
-      /* VS entry for GS->VS ring */
-      /* stride 0, num records - size, elsize0,
-         index stride 0 */
-      desc[0] = gsvs_va;
-      desc[1] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32);
-      desc[2] = gsvs_ring_size;
-      desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1);
-      } else {
-         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
-      }
-
-      /* stride gsvs_itemsize, num records 64
-         elsize 4, index stride 16 */
-      /* shader will patch stride and desc[2] */
-      desc[4] = gsvs_va;
-      desc[5] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32);
-      desc[6] = 0;
-      desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
-                S_008F0C_INDEX_STRIDE(1) | S_008F0C_ADD_TID_ENABLE(true);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11)
-         desc[5] |= S_008F04_SWIZZLE_ENABLE_GFX11(1);
-      else
-         desc[5] |= S_008F04_SWIZZLE_ENABLE_GFX6(1);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX8) {
-         /* DATA_FORMAT is STRIDE[14:17] for MUBUF with ADD_TID_ENABLE=1 */
-         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(0) | S_008F0C_ELEMENT_SIZE(1);
-      } else {
-         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) | S_008F0C_ELEMENT_SIZE(1);
-      }
-   }
-
-   desc += 8;
-
-   if (tess_rings_bo) {
-      uint64_t tess_va = radv_buffer_get_va(tess_rings_bo);
-      uint64_t tess_offchip_va = tess_va + device->physical_device->hs.tess_offchip_ring_offset;
-
-      desc[0] = tess_va;
-      desc[1] = S_008F04_BASE_ADDRESS_HI(tess_va >> 32);
-      desc[2] = device->physical_device->hs.tess_factor_ring_size;
-      desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1);
-      } else {
-         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
-      }
-
-      desc[4] = tess_offchip_va;
-      desc[5] = S_008F04_BASE_ADDRESS_HI(tess_offchip_va >> 32);
-      desc[6] = device->physical_device->hs.tess_offchip_ring_size;
-      desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW);
-      } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1);
-      } else {
-         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
-                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
-      }
-   }
-
-   desc += 8;
-
-   if (task_rings_bo) {
-      uint64_t task_va = radv_buffer_get_va(task_rings_bo);
-      uint64_t task_draw_ring_va = task_va + device->physical_device->task_info.draw_ring_offset;
-      uint64_t task_payload_ring_va = task_va + device->physical_device->task_info.payload_ring_offset;
-
-      desc[0] = task_draw_ring_va;
-      desc[1] = S_008F04_BASE_ADDRESS_HI(task_draw_ring_va >> 32);
-      desc[2] = device->physical_device->task_info.num_entries * AC_TASK_DRAW_ENTRY_BYTES;
-      desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_UINT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED);
-      } else {
-         assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_UINT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1);
-      }
-
-      desc[4] = task_payload_ring_va;
-      desc[5] = S_008F04_BASE_ADDRESS_HI(task_payload_ring_va >> 32);
-      desc[6] = device->physical_device->task_info.num_entries * AC_TASK_PAYLOAD_ENTRY_BYTES;
-      desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_UINT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED);
-      } else {
-         assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
-         desc[7] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_UINT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1);
-      }
-   }
-
-   desc += 8;
-
-   if (mesh_scratch_ring_bo) {
-      uint64_t va = radv_buffer_get_va(mesh_scratch_ring_bo);
-
-      desc[0] = va;
-      desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
-      desc[2] = RADV_MESH_SCRATCH_NUM_ENTRIES * RADV_MESH_SCRATCH_ENTRY_BYTES;
-      desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_UINT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED);
-      } else {
-         assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
-         desc[3] |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_UINT) |
-                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) | S_008F0C_RESOURCE_LEVEL(1);
-      }
-   }
-
-   desc += 4;
-
-   if (attr_ring_bo) {
-      assert(device->physical_device->rad_info.gfx_level >= GFX11);
-
-      uint64_t va = radv_buffer_get_va(attr_ring_bo);
-
-      desc[0] = va;
-      desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) | S_008F04_SWIZZLE_ENABLE_GFX11(3) /* 16B */;
-      desc[2] = attr_ring_size;
-      desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
-                S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
-                S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_32_32_32_FLOAT) |
-                S_008F0C_INDEX_STRIDE(2) /* 32 elements */;
-   }
-
-   desc += 4;
-
-   if (add_sample_positions) {
-      /* add sample positions after all rings */
-      memcpy(desc, device->sample_locations_1x, 8);
-      desc += 2;
-      memcpy(desc, device->sample_locations_2x, 16);
-      desc += 4;
-      memcpy(desc, device->sample_locations_4x, 32);
-      desc += 8;
-      memcpy(desc, device->sample_locations_8x, 64);
-   }
-}
-
-static void
-radv_emit_gs_ring_sizes(struct radv_device *device, struct radeon_cmdbuf *cs,
-                        struct radeon_winsys_bo *esgs_ring_bo, uint32_t esgs_ring_size,
-                        struct radeon_winsys_bo *gsvs_ring_bo, uint32_t gsvs_ring_size)
-{
-   if (!esgs_ring_bo && !gsvs_ring_bo)
-      return;
-
-   if (esgs_ring_bo)
-      radv_cs_add_buffer(device->ws, cs, esgs_ring_bo);
-
-   if (gsvs_ring_bo)
-      radv_cs_add_buffer(device->ws, cs, gsvs_ring_bo);
-
-   if (device->physical_device->rad_info.gfx_level >= GFX7) {
-      radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2);
-      radeon_emit(cs, esgs_ring_size >> 8);
-      radeon_emit(cs, gsvs_ring_size >> 8);
-   } else {
-      radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2);
-      radeon_emit(cs, esgs_ring_size >> 8);
-      radeon_emit(cs, gsvs_ring_size >> 8);
-   }
-}
-
-static void
-radv_emit_tess_factor_ring(struct radv_device *device, struct radeon_cmdbuf *cs,
-                           struct radeon_winsys_bo *tess_rings_bo)
-{
-   uint64_t tf_va;
-   uint32_t tf_ring_size;
-   if (!tess_rings_bo)
-      return;
-
-   tf_ring_size = device->physical_device->hs.tess_factor_ring_size / 4;
-   tf_va = radv_buffer_get_va(tess_rings_bo);
-
-   radv_cs_add_buffer(device->ws, cs, tess_rings_bo);
-
-   if (device->physical_device->rad_info.gfx_level >= GFX7) {
-      if (device->physical_device->rad_info.gfx_level >= GFX11) {
-         /* TF_RING_SIZE is per SE on GFX11. */
-         tf_ring_size /= device->physical_device->rad_info.max_se;
-      }
-
-      radeon_set_uconfig_reg(cs, R_030938_VGT_TF_RING_SIZE, S_030938_SIZE(tf_ring_size));
-      radeon_set_uconfig_reg(cs, R_030940_VGT_TF_MEMORY_BASE, tf_va >> 8);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX10) {
-         radeon_set_uconfig_reg(cs, R_030984_VGT_TF_MEMORY_BASE_HI,
-                                S_030984_BASE_HI(tf_va >> 40));
-      } else if (device->physical_device->rad_info.gfx_level == GFX9) {
-         radeon_set_uconfig_reg(cs, R_030944_VGT_TF_MEMORY_BASE_HI, S_030944_BASE_HI(tf_va >> 40));
-      }
-
-      radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM, device->physical_device->hs.hs_offchip_param);
-   } else {
-      radeon_set_config_reg(cs, R_008988_VGT_TF_RING_SIZE, S_008988_SIZE(tf_ring_size));
-      radeon_set_config_reg(cs, R_0089B8_VGT_TF_MEMORY_BASE, tf_va >> 8);
-      radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM, device->physical_device->hs.hs_offchip_param);
-   }
-}
-
-static VkResult
-radv_initialise_task_control_buffer(struct radv_device *device,
-                                    struct radeon_winsys_bo *task_rings_bo)
-{
-   uint32_t *ptr = (uint32_t *)device->ws->buffer_map(task_rings_bo);
-   if (!ptr)
-      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
-
-   const uint32_t num_entries = device->physical_device->task_info.num_entries;
-   const uint64_t task_va = radv_buffer_get_va(task_rings_bo);
-   const uint64_t task_draw_ring_va = task_va + device->physical_device->task_info.draw_ring_offset;
-   assert((task_draw_ring_va & 0xFFFFFF00) == (task_draw_ring_va & 0xFFFFFFFF));
-
-   /* 64-bit write_ptr */
-   ptr[0] = num_entries;
-   ptr[1] = 0;
-   /* 64-bit read_ptr */
-   ptr[2] = num_entries;
-   ptr[3] = 0;
-   /* 64-bit dealloc_ptr */
-   ptr[4] = num_entries;
-   ptr[5] = 0;
-   /* num_entries */
-   ptr[6] = num_entries;
-   /* 64-bit draw ring address */
-   ptr[7] = task_draw_ring_va;
-   ptr[8] = task_draw_ring_va >> 32;
-
-   device->ws->buffer_unmap(task_rings_bo);
-   return VK_SUCCESS;
-}
-
-static void
-radv_emit_task_rings(struct radv_device *device, struct radeon_cmdbuf *cs,
-                     struct radeon_winsys_bo *task_rings_bo, bool compute)
-{
-   if (!task_rings_bo)
-      return;
-
-   const uint64_t task_ctrlbuf_va = radv_buffer_get_va(task_rings_bo);
-   assert(radv_is_aligned(task_ctrlbuf_va, 256));
-   radv_cs_add_buffer(device->ws, cs, task_rings_bo);
-
-   /* Tell the GPU where the task control buffer is. */
-   radeon_emit(cs, PKT3(PKT3_DISPATCH_TASK_STATE_INIT, 1, 0) | PKT3_SHADER_TYPE_S(!!compute));
-   /* bits [31:8]: control buffer address lo, bits[7:0]: reserved (set to zero) */
-   radeon_emit(cs, task_ctrlbuf_va & 0xFFFFFF00);
-   /* bits [31:0]: control buffer address hi */
-   radeon_emit(cs, task_ctrlbuf_va >> 32);
-}
-
-static void
-radv_emit_graphics_scratch(struct radv_device *device, struct radeon_cmdbuf *cs,
-                           uint32_t size_per_wave, uint32_t waves,
-                           struct radeon_winsys_bo *scratch_bo)
-{
-   struct radeon_info *info = &device->physical_device->rad_info;
-
-   if (!scratch_bo)
-      return;
-
-   radv_cs_add_buffer(device->ws, cs, scratch_bo);
-
-   if (info->gfx_level >= GFX11) {
-      uint64_t va = radv_buffer_get_va(scratch_bo);
-
-      /* WAVES is per SE for SPI_TMPRING_SIZE. */
-      waves /= info->num_se;
-
-      radeon_set_context_reg_seq(cs, R_0286E8_SPI_TMPRING_SIZE, 3);
-      radeon_emit(cs, S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(round_up_u32(size_per_wave, 256)));
-      radeon_emit(cs, va >> 8);  /* SPI_GFX_SCRATCH_BASE_LO */
-      radeon_emit(cs, va >> 40); /* SPI_GFX_SCRATCH_BASE_HI */
-   } else {
-      radeon_set_context_reg(
-         cs, R_0286E8_SPI_TMPRING_SIZE,
-         S_0286E8_WAVES(waves) | S_0286E8_WAVESIZE(round_up_u32(size_per_wave, 1024)));
-   }
-}
-
-static void
-radv_emit_compute_scratch(struct radv_device *device, struct radeon_cmdbuf *cs,
-                          uint32_t size_per_wave, uint32_t waves,
-                          struct radeon_winsys_bo *compute_scratch_bo)
-{
-   struct radeon_info *info = &device->physical_device->rad_info;
-   uint64_t scratch_va;
-   uint32_t rsrc1;
-
-   if (!compute_scratch_bo)
-      return;
-
-   scratch_va = radv_buffer_get_va(compute_scratch_bo);
-   rsrc1 = S_008F04_BASE_ADDRESS_HI(scratch_va >> 32);
-
-   if (info->gfx_level >= GFX11)
-      rsrc1 |= S_008F04_SWIZZLE_ENABLE_GFX11(1);
-   else
-      rsrc1 |= S_008F04_SWIZZLE_ENABLE_GFX6(1);
-
-   radv_cs_add_buffer(device->ws, cs, compute_scratch_bo);
-
-   if (info->gfx_level >= GFX11) {
-      radeon_set_sh_reg_seq(cs, R_00B840_COMPUTE_DISPATCH_SCRATCH_BASE_LO, 2);
-      radeon_emit(cs, scratch_va >> 8);
-      radeon_emit(cs, scratch_va >> 40);
-
-      waves /= info->num_se;
-   }
-
-   radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2);
-   radeon_emit(cs, scratch_va);
-   radeon_emit(cs, rsrc1);
-
-   radeon_set_sh_reg(cs, R_00B860_COMPUTE_TMPRING_SIZE,
-                     S_00B860_WAVES(waves) |
-                     S_00B860_WAVESIZE(round_up_u32(size_per_wave, info->gfx_level >= GFX11 ? 256 : 1024)));
-}
-
-static void
-radv_emit_compute_shader_pointers(struct radv_device *device, struct radeon_cmdbuf *cs,
-                                  struct radeon_winsys_bo *descriptor_bo)
-{
-   if (!descriptor_bo)
-      return;
-
-   uint64_t va = radv_buffer_get_va(descriptor_bo);
-   radv_cs_add_buffer(device->ws, cs, descriptor_bo);
-
-   /* Compute shader user data 0-1 have the scratch pointer (unlike GFX shaders),
-    * so emit the descriptor pointer to user data 2-3 instead (task_ring_offsets arg).
-    */
-   radv_emit_shader_pointer(device, cs, R_00B908_COMPUTE_USER_DATA_2, va, true);
-}
-
-static void
-radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmdbuf *cs,
-                                   struct radeon_winsys_bo *descriptor_bo)
-{
-   uint64_t va;
-
-   if (!descriptor_bo)
-      return;
-
-   va = radv_buffer_get_va(descriptor_bo);
-
-   radv_cs_add_buffer(device->ws, cs, descriptor_bo);
-
-   if (device->physical_device->rad_info.gfx_level >= GFX11) {
-      uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
-                         R_00B420_SPI_SHADER_PGM_LO_HS,
-                         R_00B220_SPI_SHADER_PGM_LO_GS};
-
-      for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
-         radv_emit_shader_pointer(device, cs, regs[i], va, true);
-      }
-   } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
-      uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0,
-                         R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
-                         R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
-
-      for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
-         radv_emit_shader_pointer(device, cs, regs[i], va, true);
-      }
-   } else if (device->physical_device->rad_info.gfx_level == GFX9) {
-      uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0,
-                         R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
-                         R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
-
-      for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
-         radv_emit_shader_pointer(device, cs, regs[i], va, true);
-      }
-   } else {
-      uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0, R_00B130_SPI_SHADER_USER_DATA_VS_0,
-                         R_00B230_SPI_SHADER_USER_DATA_GS_0, R_00B330_SPI_SHADER_USER_DATA_ES_0,
-                         R_00B430_SPI_SHADER_USER_DATA_HS_0, R_00B530_SPI_SHADER_USER_DATA_LS_0};
-
-      for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
-         radv_emit_shader_pointer(device, cs, regs[i], va, true);
-      }
-   }
-}
-
-static void
-radv_emit_attribute_ring(struct radv_device *device, struct radeon_cmdbuf *cs,
-                         struct radeon_winsys_bo *attr_ring_bo, uint32_t attr_ring_size)
-{
-   const struct radv_physical_device *pdevice = device->physical_device;
-   uint64_t va;
-
-   if (!attr_ring_bo)
-      return;
-
-   assert(pdevice->rad_info.gfx_level >= GFX11);
-
-   va = radv_buffer_get_va(attr_ring_bo);
-   assert((va >> 32) == pdevice->rad_info.address32_hi);
-
-   radv_cs_add_buffer(device->ws, cs, attr_ring_bo);
-
-   /* We must wait for idle using an EOP event before changing the attribute ring registers. Use the
-    * bottom-of-pipe EOP event, but increment the PWS counter instead of writing memory.
-    */
-   radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0));
-   radeon_emit(cs, S_490_EVENT_TYPE(V_028A90_BOTTOM_OF_PIPE_TS) | S_490_EVENT_INDEX(5) |
-                   S_490_PWS_ENABLE(1));
-   radeon_emit(cs, 0); /* DST_SEL, INT_SEL, DATA_SEL */
-   radeon_emit(cs, 0); /* ADDRESS_LO */
-   radeon_emit(cs, 0); /* ADDRESS_HI */
-   radeon_emit(cs, 0); /* DATA_LO */
-   radeon_emit(cs, 0); /* DATA_HI */
-   radeon_emit(cs, 0); /* INT_CTXID */
-
-   /* Wait for the PWS counter. */
-   radeon_emit(cs, PKT3(PKT3_ACQUIRE_MEM, 6, 0));
-   radeon_emit(cs, S_580_PWS_STAGE_SEL(V_580_CP_ME) |
-                   S_580_PWS_COUNTER_SEL(V_580_TS_SELECT) |
-                   S_580_PWS_ENA2(1) | S_580_PWS_COUNT(0));
-   radeon_emit(cs, 0xffffffff); /* GCR_SIZE */
-   radeon_emit(cs, 0x01ffffff); /* GCR_SIZE_HI */
-   radeon_emit(cs, 0); /* GCR_BASE_LO */
-   radeon_emit(cs, 0); /* GCR_BASE_HI */
-   radeon_emit(cs, S_585_PWS_ENA(1));
-   radeon_emit(cs, 0); /* GCR_CNTL */
-
-   /* The PS will read inputs from this address. */
-   radeon_set_uconfig_reg(cs, R_031118_SPI_ATTRIBUTE_RING_BASE, va >> 16);
-   radeon_set_uconfig_reg(cs, R_03111C_SPI_ATTRIBUTE_RING_SIZE,
-                              S_03111C_MEM_SIZE(((attr_ring_size / pdevice->rad_info.max_se) >> 16) - 1) |
-                              S_03111C_BIG_PAGE(pdevice->rad_info.discardable_allows_big_page) |
-                              S_03111C_L1_POLICY(1));
-}
-
-static void
-radv_init_graphics_state(struct radeon_cmdbuf *cs, struct radv_device *device)
-{
-   if (device->gfx_init) {
-      uint64_t va = radv_buffer_get_va(device->gfx_init);
-
-      radeon_emit(cs, PKT3(PKT3_INDIRECT_BUFFER_CIK, 2, 0));
-      radeon_emit(cs, va);
-      radeon_emit(cs, va >> 32);
-      radeon_emit(cs, device->gfx_init_size_dw & 0xffff);
-
-      radv_cs_add_buffer(device->ws, cs, device->gfx_init);
-   } else {
-      si_emit_graphics(device, cs);
-   }
-}
-
-static void
-radv_init_compute_state(struct radeon_cmdbuf *cs, struct radv_device *device)
-{
-   si_emit_compute(device, cs);
-}
-
-static VkResult
-radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *device,
-                        const struct radv_queue_ring_info *needs)
-{
-   struct radeon_winsys *ws = device->ws;
-   struct radeon_winsys_bo *scratch_bo = queue->scratch_bo;
-   struct radeon_winsys_bo *descriptor_bo = queue->descriptor_bo;
-   struct radeon_winsys_bo *compute_scratch_bo = queue->compute_scratch_bo;
-   struct radeon_winsys_bo *esgs_ring_bo = queue->esgs_ring_bo;
-   struct radeon_winsys_bo *gsvs_ring_bo = queue->gsvs_ring_bo;
-   struct radeon_winsys_bo *tess_rings_bo = queue->tess_rings_bo;
-   struct radeon_winsys_bo *task_rings_bo = queue->task_rings_bo;
-   struct radeon_winsys_bo *mesh_scratch_ring_bo = queue->mesh_scratch_ring_bo;
-   struct radeon_winsys_bo *attr_ring_bo = queue->attr_ring_bo;
-   struct radeon_winsys_bo *gds_bo = queue->gds_bo;
-   struct radeon_winsys_bo *gds_oa_bo = queue->gds_oa_bo;
-   struct radeon_cmdbuf *dest_cs[3] = {0};
-   const uint32_t ring_bo_flags = RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING;
-   VkResult result = VK_SUCCESS;
-
-   const bool add_sample_positions = !queue->ring_info.sample_positions && needs->sample_positions;
-   const uint32_t scratch_size = needs->scratch_size_per_wave * needs->scratch_waves;
-   const uint32_t queue_scratch_size =
-      queue->ring_info.scratch_size_per_wave * queue->ring_info.scratch_waves;
-
-   if (scratch_size > queue_scratch_size) {
-      result = ws->buffer_create(ws, scratch_size, 4096, RADEON_DOMAIN_VRAM, ring_bo_flags,
-                                 RADV_BO_PRIORITY_SCRATCH, 0, &scratch_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(device, scratch_bo, 0, 0, scratch_size);
-   }
-
-   const uint32_t compute_scratch_size =
-      needs->compute_scratch_size_per_wave * needs->compute_scratch_waves;
-   const uint32_t compute_queue_scratch_size =
-      queue->ring_info.compute_scratch_size_per_wave * queue->ring_info.compute_scratch_waves;
-   if (compute_scratch_size > compute_queue_scratch_size) {
-      result = ws->buffer_create(ws, compute_scratch_size, 4096, RADEON_DOMAIN_VRAM, ring_bo_flags,
-                                 RADV_BO_PRIORITY_SCRATCH, 0, &compute_scratch_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(device, compute_scratch_bo, 0, 0, compute_scratch_size);
-   }
-
-   if (needs->esgs_ring_size > queue->ring_info.esgs_ring_size) {
-      result = ws->buffer_create(ws, needs->esgs_ring_size, 4096, RADEON_DOMAIN_VRAM, ring_bo_flags,
-                                 RADV_BO_PRIORITY_SCRATCH, 0, &esgs_ring_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(device, esgs_ring_bo, 0, 0, needs->esgs_ring_size);
-   }
-
-   if (needs->gsvs_ring_size > queue->ring_info.gsvs_ring_size) {
-      result = ws->buffer_create(ws, needs->gsvs_ring_size, 4096, RADEON_DOMAIN_VRAM, ring_bo_flags,
-                                 RADV_BO_PRIORITY_SCRATCH, 0, &gsvs_ring_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(device, gsvs_ring_bo, 0, 0, needs->gsvs_ring_size);
-   }
-
-   if (!queue->ring_info.tess_rings && needs->tess_rings) {
-      uint64_t tess_rings_size = device->physical_device->hs.tess_offchip_ring_offset +
-                                 device->physical_device->hs.tess_offchip_ring_size;
-      result = ws->buffer_create(ws, tess_rings_size, 256, RADEON_DOMAIN_VRAM, ring_bo_flags,
-                                 RADV_BO_PRIORITY_SCRATCH, 0, &tess_rings_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(device, tess_rings_bo, 0, 0, tess_rings_size);
-   }
-
-   if (!queue->ring_info.task_rings && needs->task_rings) {
-      assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
-
-      /* We write the control buffer from the CPU, so need to grant CPU access to the BO.
-       * The draw ring needs to be zero-initialized otherwise the ready bits will be incorrect.
-       */
-      uint32_t task_rings_bo_flags =
-         RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_ZERO_VRAM;
-
-      result = ws->buffer_create(ws, device->physical_device->task_info.bo_size_bytes, 256,
-                                 RADEON_DOMAIN_VRAM, task_rings_bo_flags, RADV_BO_PRIORITY_SCRATCH,
-                                 0, &task_rings_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(device, task_rings_bo, 0, 0,
-                                            device->physical_device->task_info.bo_size_bytes);
-
-      result = radv_initialise_task_control_buffer(device, task_rings_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-   }
-
-   if (!queue->ring_info.mesh_scratch_ring && needs->mesh_scratch_ring) {
-      assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
-      result =
-         ws->buffer_create(ws, RADV_MESH_SCRATCH_NUM_ENTRIES * RADV_MESH_SCRATCH_ENTRY_BYTES, 256,
-                           RADEON_DOMAIN_VRAM, ring_bo_flags, RADV_BO_PRIORITY_SCRATCH, 0, &mesh_scratch_ring_bo);
-
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(
-         device, mesh_scratch_ring_bo, 0, 0,
-         RADV_MESH_SCRATCH_NUM_ENTRIES * RADV_MESH_SCRATCH_ENTRY_BYTES);
-   }
-
-   if (needs->attr_ring_size > queue->ring_info.attr_ring_size) {
-      assert(device->physical_device->rad_info.gfx_level >= GFX11);
-      result = ws->buffer_create(ws, needs->attr_ring_size, 2 * 1024 * 1024 /* 2MiB */,
-                                 RADEON_DOMAIN_VRAM, RADEON_FLAG_32BIT | RADEON_FLAG_DISCARDABLE |
-                                 ring_bo_flags, RADV_BO_PRIORITY_SCRATCH, 0, &attr_ring_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-      radv_rmv_log_command_buffer_bo_create(device, attr_ring_bo, 0, 0, needs->attr_ring_size);
-   }
-
-   if (!queue->ring_info.gds && needs->gds) {
-      assert(device->physical_device->rad_info.gfx_level >= GFX10);
-
-      /* 4 streamout GDS counters.
-       * We need 256B (64 dw) of GDS, otherwise streamout hangs.
-       */
-      result = ws->buffer_create(ws, 256, 4, RADEON_DOMAIN_GDS, ring_bo_flags,
-                                 RADV_BO_PRIORITY_SCRATCH, 0, &gds_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-
-      /* Add the GDS BO to our global BO list to prevent the kernel to emit a GDS switch and reset
-       * the state when a compute queue is used.
-       */
-      result = device->ws->buffer_make_resident(ws, gds_bo, true);
-      if (result != VK_SUCCESS)
-         goto fail;
-   }
-
-   if (!queue->ring_info.gds_oa && needs->gds_oa) {
-      assert(device->physical_device->rad_info.gfx_level >= GFX10);
-
-      result = ws->buffer_create(ws, 4, 1, RADEON_DOMAIN_OA, ring_bo_flags,
-                                 RADV_BO_PRIORITY_SCRATCH, 0, &gds_oa_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-
-      /* Add the GDS OA BO to our global BO list to prevent the kernel to emit a GDS switch and
-       * reset the state when a compute queue is used.
-       */
-      result = device->ws->buffer_make_resident(ws, gds_oa_bo, true);
-      if (result != VK_SUCCESS)
-         goto fail;
-   }
-
-   /* Re-initialize the descriptor BO when any ring BOs changed.
-    *
-    * Additionally, make sure to create the descriptor BO for the compute queue
-    * when it uses the task shader rings. The task rings BO is shared between the
-    * GFX and compute queues and already initialized here.
-    */
-   if ((queue->qf == RADV_QUEUE_COMPUTE && !descriptor_bo && task_rings_bo) ||
-       scratch_bo != queue->scratch_bo || esgs_ring_bo != queue->esgs_ring_bo ||
-       gsvs_ring_bo != queue->gsvs_ring_bo || tess_rings_bo != queue->tess_rings_bo ||
-       task_rings_bo != queue->task_rings_bo || mesh_scratch_ring_bo != queue->mesh_scratch_ring_bo ||
-       attr_ring_bo != queue->attr_ring_bo || add_sample_positions) {
-      uint32_t size = 0;
-      if (gsvs_ring_bo || esgs_ring_bo || tess_rings_bo || task_rings_bo || mesh_scratch_ring_bo ||
-          attr_ring_bo || add_sample_positions) {
-         size = 176; /* 2 dword + 2 padding + 4 dword * 10 */
-         if (add_sample_positions)
-            size += 128; /* 64+32+16+8 = 120 bytes */
-      } else if (scratch_bo) {
-         size = 8; /* 2 dword */
-      }
-
-      result = ws->buffer_create(
-         ws, size, 4096, RADEON_DOMAIN_VRAM,
-         RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_READ_ONLY,
-         RADV_BO_PRIORITY_DESCRIPTOR, 0, &descriptor_bo);
-      if (result != VK_SUCCESS)
-         goto fail;
-   }
-
-   if (descriptor_bo != queue->descriptor_bo) {
-      uint32_t *map = (uint32_t *)ws->buffer_map(descriptor_bo);
-      if (!map)
-         goto fail;
-
-      if (scratch_bo) {
-         uint64_t scratch_va = radv_buffer_get_va(scratch_bo);
-         uint32_t rsrc1 = S_008F04_BASE_ADDRESS_HI(scratch_va >> 32);
-
-         if (device->physical_device->rad_info.gfx_level >= GFX11)
-            rsrc1 |= S_008F04_SWIZZLE_ENABLE_GFX11(1);
-         else
-            rsrc1 |= S_008F04_SWIZZLE_ENABLE_GFX6(1);
-
-         map[0] = scratch_va;
-         map[1] = rsrc1;
-      }
-
-      if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || task_rings_bo || mesh_scratch_ring_bo ||
-          attr_ring_bo || add_sample_positions)
-         radv_fill_shader_rings(device, map, add_sample_positions, needs->esgs_ring_size,
-                                esgs_ring_bo, needs->gsvs_ring_size, gsvs_ring_bo, tess_rings_bo,
-                                task_rings_bo, mesh_scratch_ring_bo, needs->attr_ring_size,
-                                attr_ring_bo);
-
-      ws->buffer_unmap(descriptor_bo);
-   }
-
-   for (int i = 0; i < 3; ++i) {
-      /* Don't create continue preamble when it's not necessary. */
-      if (i == 2) {
-         /* We only need the continue preamble when we can't use indirect buffers. */
-         if (!(device->instance->debug_flags & RADV_DEBUG_NO_IBS) &&
-             device->physical_device->rad_info.gfx_level >= GFX7)
-            continue;
-         /* Continue preamble is unnecessary when no shader rings are used. */
-         if (!needs->scratch_size_per_wave && !needs->compute_scratch_size_per_wave &&
-             !needs->esgs_ring_size && !needs->gsvs_ring_size && !needs->tess_rings &&
-             !needs->task_rings && !needs->mesh_scratch_ring && !needs->attr_ring_size &&
-             !needs->gds && !needs->gds_oa && !needs->sample_positions)
-            continue;
-      }
-
-      enum rgp_flush_bits sqtt_flush_bits = 0;
-      struct radeon_cmdbuf *cs = NULL;
-      cs = ws->cs_create(ws, radv_queue_family_to_ring(device->physical_device, queue->qf));
-      if (!cs) {
-         result = VK_ERROR_OUT_OF_HOST_MEMORY;
-         goto fail;
-      }
-
-      dest_cs[i] = cs;
-
-      if (scratch_bo)
-         radv_cs_add_buffer(ws, cs, scratch_bo);
-
-      /* Emit initial configuration. */
-      switch (queue->qf) {
-      case RADV_QUEUE_GENERAL:
-         if (queue->uses_shadow_regs)
-            radv_emit_shadow_regs_preamble(cs, device, queue);
-         radv_init_graphics_state(cs, device);
-
-         if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || task_rings_bo) {
-            radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
-            radeon_emit(cs, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
-
-            radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
-            radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
-         }
-
-         radv_emit_gs_ring_sizes(device, cs, esgs_ring_bo, needs->esgs_ring_size, gsvs_ring_bo,
-                                 needs->gsvs_ring_size);
-         radv_emit_tess_factor_ring(device, cs, tess_rings_bo);
-         radv_emit_task_rings(device, cs, task_rings_bo, false);
-         radv_emit_attribute_ring(device, cs, attr_ring_bo, needs->attr_ring_size);
-         radv_emit_graphics_shader_pointers(device, cs, descriptor_bo);
-         radv_emit_compute_scratch(device, cs, needs->compute_scratch_size_per_wave,
-                                   needs->compute_scratch_waves, compute_scratch_bo);
-         radv_emit_graphics_scratch(device, cs, needs->scratch_size_per_wave, needs->scratch_waves,
-                                    scratch_bo);
-         break;
-      case RADV_QUEUE_COMPUTE:
-         radv_init_compute_state(cs, device);
-
-         if (task_rings_bo) {
-            radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
-            radeon_emit(cs, EVENT_TYPE(V_028A90_CS_PARTIAL_FLUSH) | EVENT_INDEX(4));
-         }
-
-         radv_emit_task_rings(device, cs, task_rings_bo, true);
-         radv_emit_compute_shader_pointers(device, cs, descriptor_bo);
-         radv_emit_compute_scratch(device, cs, needs->compute_scratch_size_per_wave,
-                                   needs->compute_scratch_waves, compute_scratch_bo);
-         break;
-      default:
-         break;
-      }
-
-      if (i < 2) {
-         /* The two initial preambles have a cache flush at the beginning. */
-         const enum amd_gfx_level gfx_level = device->physical_device->rad_info.gfx_level;
-         const bool is_mec = queue->qf == RADV_QUEUE_COMPUTE && gfx_level >= GFX7;
-         enum radv_cmd_flush_bits flush_bits = RADV_CMD_FLAG_INV_ICACHE | RADV_CMD_FLAG_INV_SCACHE |
-                                               RADV_CMD_FLAG_INV_VCACHE | RADV_CMD_FLAG_INV_L2 |
-                                               RADV_CMD_FLAG_START_PIPELINE_STATS;
-
-         if (i == 0) {
-            /* The full flush preamble should also wait for previous shader work to finish. */
-            flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH;
-            if (queue->qf == RADV_QUEUE_GENERAL)
-               flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH;
-         }
-
-         si_cs_emit_cache_flush(cs, gfx_level, NULL, 0, is_mec, flush_bits, &sqtt_flush_bits, 0);
-      }
-
-      result = ws->cs_finalize(cs);
-      if (result != VK_SUCCESS)
-         goto fail;
-   }
-
-   if (queue->initial_full_flush_preamble_cs)
-      ws->cs_destroy(queue->initial_full_flush_preamble_cs);
-
-   if (queue->initial_preamble_cs)
-      ws->cs_destroy(queue->initial_preamble_cs);
-
-   if (queue->continue_preamble_cs)
-      ws->cs_destroy(queue->continue_preamble_cs);
-
-   queue->initial_full_flush_preamble_cs = dest_cs[0];
-   queue->initial_preamble_cs = dest_cs[1];
-   queue->continue_preamble_cs = dest_cs[2];
-
-   if (scratch_bo != queue->scratch_bo) {
-      if (queue->scratch_bo) {
-         ws->buffer_destroy(ws, queue->scratch_bo);
-         radv_rmv_log_command_buffer_bo_destroy(device, queue->scratch_bo);
-      }
-      queue->scratch_bo = scratch_bo;
-   }
-
-   if (compute_scratch_bo != queue->compute_scratch_bo) {
-      if (queue->compute_scratch_bo) {
-         ws->buffer_destroy(ws, queue->compute_scratch_bo);
-         radv_rmv_log_command_buffer_bo_destroy(device, queue->compute_scratch_bo);
-      }
-      queue->compute_scratch_bo = compute_scratch_bo;
-   }
-
-   if (esgs_ring_bo != queue->esgs_ring_bo) {
-      if (queue->esgs_ring_bo) {
-         ws->buffer_destroy(ws, queue->esgs_ring_bo);
-         radv_rmv_log_command_buffer_bo_destroy(device, queue->esgs_ring_bo);
-      }
-      queue->esgs_ring_bo = esgs_ring_bo;
-   }
-
-   if (gsvs_ring_bo != queue->gsvs_ring_bo) {
-      if (queue->gsvs_ring_bo) {
-         ws->buffer_destroy(ws, queue->gsvs_ring_bo);
-         radv_rmv_log_command_buffer_bo_destroy(device, queue->gsvs_ring_bo);
-      }
-      queue->gsvs_ring_bo = gsvs_ring_bo;
-   }
-
-   if (descriptor_bo != queue->descriptor_bo) {
-      if (queue->descriptor_bo)
-         ws->buffer_destroy(ws, queue->descriptor_bo);
-      queue->descriptor_bo = descriptor_bo;
-   }
-
-   queue->tess_rings_bo = tess_rings_bo;
-   queue->task_rings_bo = task_rings_bo;
-   queue->mesh_scratch_ring_bo = mesh_scratch_ring_bo;
-   queue->attr_ring_bo = attr_ring_bo;
-   queue->gds_bo = gds_bo;
-   queue->gds_oa_bo = gds_oa_bo;
-   queue->ring_info = *needs;
-   return VK_SUCCESS;
-fail:
-   for (int i = 0; i < ARRAY_SIZE(dest_cs); ++i)
-      if (dest_cs[i])
-         ws->cs_destroy(dest_cs[i]);
-   if (descriptor_bo && descriptor_bo != queue->descriptor_bo)
-      ws->buffer_destroy(ws, descriptor_bo);
-   if (scratch_bo && scratch_bo != queue->scratch_bo)
-      ws->buffer_destroy(ws, scratch_bo);
-   if (compute_scratch_bo && compute_scratch_bo != queue->compute_scratch_bo)
-      ws->buffer_destroy(ws, compute_scratch_bo);
-   if (esgs_ring_bo && esgs_ring_bo != queue->esgs_ring_bo)
-      ws->buffer_destroy(ws, esgs_ring_bo);
-   if (gsvs_ring_bo && gsvs_ring_bo != queue->gsvs_ring_bo)
-      ws->buffer_destroy(ws, gsvs_ring_bo);
-   if (tess_rings_bo && tess_rings_bo != queue->tess_rings_bo)
-      ws->buffer_destroy(ws, tess_rings_bo);
-   if (task_rings_bo && task_rings_bo != queue->task_rings_bo)
-      ws->buffer_destroy(ws, task_rings_bo);
-   if (attr_ring_bo && attr_ring_bo != queue->attr_ring_bo)
-      ws->buffer_destroy(ws, attr_ring_bo);
-   if (gds_bo && gds_bo != queue->gds_bo) {
-      ws->buffer_make_resident(ws, queue->gds_bo, false);
-      ws->buffer_destroy(ws, gds_bo);
-   }
-   if (gds_oa_bo && gds_oa_bo != queue->gds_oa_bo) {
-      ws->buffer_make_resident(ws, queue->gds_oa_bo, false);
-      ws->buffer_destroy(ws, gds_oa_bo);
-   }
-
-   return vk_error(queue, result);
-}
-
-static struct radeon_cmdbuf *
-radv_create_perf_counter_lock_cs(struct radv_device *device, unsigned pass, bool unlock)
-{
-   struct radeon_cmdbuf **cs_ref = &device->perf_counter_lock_cs[pass * 2 + (unlock ? 1 : 0)];
-   struct radeon_cmdbuf *cs;
-
-   if (*cs_ref)
-      return *cs_ref;
-
-   cs = device->ws->cs_create(device->ws, AMD_IP_GFX);
-   if (!cs)
-      return NULL;
-
-   ASSERTED unsigned cdw = radeon_check_space(device->ws, cs, 21);
-
-   if (!unlock) {
-      uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET;
-      radeon_emit(cs, PKT3(PKT3_ATOMIC_MEM, 7, 0));
-      radeon_emit(cs, ATOMIC_OP(TC_OP_ATOMIC_CMPSWAP_32) | ATOMIC_COMMAND(ATOMIC_COMMAND_LOOP));
-      radeon_emit(cs, mutex_va);       /* addr lo */
-      radeon_emit(cs, mutex_va >> 32); /* addr hi */
-      radeon_emit(cs, 1);              /* data lo */
-      radeon_emit(cs, 0);              /* data hi */
-      radeon_emit(cs, 0);              /* compare data lo */
-      radeon_emit(cs, 0);              /* compare data hi */
-      radeon_emit(cs, 10);             /* loop interval */
-   }
-
-   uint64_t va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET;
-   uint64_t unset_va = va + (unlock ? 8 * pass : 0);
-   uint64_t set_va = va + (unlock ? 0 : 8 * pass);
-
-   radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
-   radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
-                      COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM);
-   radeon_emit(cs, 0); /* immediate */
-   radeon_emit(cs, 0);
-   radeon_emit(cs, unset_va);
-   radeon_emit(cs, unset_va >> 32);
-
-   radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
-   radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
-                      COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM);
-   radeon_emit(cs, 1); /* immediate */
-   radeon_emit(cs, 0);
-   radeon_emit(cs, set_va);
-   radeon_emit(cs, set_va >> 32);
-
-   if (unlock) {
-      uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET;
-
-      radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
-      radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
-                         COPY_DATA_COUNT_SEL | COPY_DATA_WR_CONFIRM);
-      radeon_emit(cs, 0); /* immediate */
-      radeon_emit(cs, 0);
-      radeon_emit(cs, mutex_va);
-      radeon_emit(cs, mutex_va >> 32);
-   }
-
-   assert(cs->cdw <= cdw);
-
-   VkResult result = device->ws->cs_finalize(cs);
-   if (result != VK_SUCCESS) {
-      device->ws->cs_destroy(cs);
-      return NULL;
-   }
-
-   /* All the casts are to avoid MSVC errors around pointer truncation in a non-taken
-    * alternative.
-    */
-   if (p_atomic_cmpxchg((uintptr_t*)cs_ref, 0, (uintptr_t)cs) != 0) {
-      device->ws->cs_destroy(cs);
-   }
-
-   return *cs_ref;
-}
-
-static VkResult
-radv_sparse_buffer_bind_memory(struct radv_device *device, const VkSparseBufferMemoryBindInfo *bind)
-{
-   RADV_FROM_HANDLE(radv_buffer, buffer, bind->buffer);
-   VkResult result = VK_SUCCESS;
-
-   struct radv_device_memory *mem = NULL;
-   VkDeviceSize resourceOffset = 0;
-   VkDeviceSize size = 0;
-   VkDeviceSize memoryOffset = 0;
-   for (uint32_t i = 0; i < bind->bindCount; ++i) {
-      struct radv_device_memory *cur_mem = NULL;
-
-      if (bind->pBinds[i].memory != VK_NULL_HANDLE)
-         cur_mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
-      if (i && mem == cur_mem) {
-         if (mem) {
-            if (bind->pBinds[i].resourceOffset == resourceOffset + size &&
-                bind->pBinds[i].memoryOffset == memoryOffset + size) {
-               size += bind->pBinds[i].size;
-               continue;
-            }
-         } else {
-            if (bind->pBinds[i].resourceOffset == resourceOffset + size) {
-               size += bind->pBinds[i].size;
-               continue;
-            }
-         }
-      }
-      if (size) {
-         result = device->ws->buffer_virtual_bind(device->ws, buffer->bo,
-                                                  resourceOffset, size,
-                                                  mem ? mem->bo : NULL, memoryOffset);
-         if (result != VK_SUCCESS)
-            return result;
-
-         if (bind->pBinds[i].memory)
-            radv_rmv_log_sparse_add_residency(device, buffer->bo, memoryOffset);
-         else
-            radv_rmv_log_sparse_remove_residency(device, buffer->bo, memoryOffset);
-      }
-      mem = cur_mem;
-      resourceOffset = bind->pBinds[i].resourceOffset;
-      size = bind->pBinds[i].size;
-      memoryOffset = bind->pBinds[i].memoryOffset;
-   }
-   if (size) {
-      result = device->ws->buffer_virtual_bind(device->ws, buffer->bo,
-                                               resourceOffset, size,
-                                               mem ? mem->bo : NULL, memoryOffset);
-
-      if (mem)
-         radv_rmv_log_sparse_add_residency(device, buffer->bo, memoryOffset);
-      else
-         radv_rmv_log_sparse_remove_residency(device, buffer->bo, memoryOffset);
-   }
-
-   return result;
-}
-
-static VkResult
-radv_sparse_image_opaque_bind_memory(struct radv_device *device,
-                                     const VkSparseImageOpaqueMemoryBindInfo *bind)
-{
-   RADV_FROM_HANDLE(radv_image, image, bind->image);
-   VkResult result;
-
-   for (uint32_t i = 0; i < bind->bindCount; ++i) {
-      struct radv_device_memory *mem = NULL;
-
-      if (bind->pBinds[i].memory != VK_NULL_HANDLE)
-         mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
-
-      result = device->ws->buffer_virtual_bind(device->ws, image->bindings[0].bo,
-                                               bind->pBinds[i].resourceOffset, bind->pBinds[i].size,
-                                               mem ? mem->bo : NULL, bind->pBinds[i].memoryOffset);
-      if (result != VK_SUCCESS)
-         return result;
-
-      if (bind->pBinds[i].memory)
-         radv_rmv_log_sparse_add_residency(device, image->bindings[0].bo,
-                                           bind->pBinds[i].resourceOffset);
-      else
-         radv_rmv_log_sparse_remove_residency(device, image->bindings[0].bo,
-                                              bind->pBinds[i].resourceOffset);
-   }
-
-   return VK_SUCCESS;
-}
-
-static VkResult
-radv_sparse_image_bind_memory(struct radv_device *device, const VkSparseImageMemoryBindInfo *bind)
-{
-   RADV_FROM_HANDLE(radv_image, image, bind->image);
-   struct radeon_surf *surface = &image->planes[0].surface;
-   uint32_t bs = vk_format_get_blocksize(image->vk.format);
-   VkResult result;
-
-   for (uint32_t i = 0; i < bind->bindCount; ++i) {
-      struct radv_device_memory *mem = NULL;
-      uint32_t offset, pitch, depth_pitch;
-      uint32_t mem_offset = bind->pBinds[i].memoryOffset;
-      const uint32_t layer = bind->pBinds[i].subresource.arrayLayer;
-      const uint32_t level = bind->pBinds[i].subresource.mipLevel;
-
-      VkExtent3D bind_extent = bind->pBinds[i].extent;
-      bind_extent.width =
-         DIV_ROUND_UP(bind_extent.width, vk_format_get_blockwidth(image->vk.format));
-      bind_extent.height =
-         DIV_ROUND_UP(bind_extent.height, vk_format_get_blockheight(image->vk.format));
-
-      VkOffset3D bind_offset = bind->pBinds[i].offset;
-      bind_offset.x /= vk_format_get_blockwidth(image->vk.format);
-      bind_offset.y /= vk_format_get_blockheight(image->vk.format);
-
-      if (bind->pBinds[i].memory != VK_NULL_HANDLE)
-         mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
-
-      if (device->physical_device->rad_info.gfx_level >= GFX9) {
-         offset = surface->u.gfx9.surf_slice_size * layer + surface->u.gfx9.prt_level_offset[level];
-         pitch = surface->u.gfx9.prt_level_pitch[level];
-         depth_pitch = surface->u.gfx9.surf_slice_size;
-      } else {
-         depth_pitch = surface->u.legacy.level[level].slice_size_dw * 4;
-         offset = (uint64_t)surface->u.legacy.level[level].offset_256B * 256 + depth_pitch * layer;
-         pitch = surface->u.legacy.level[level].nblk_x;
-      }
-
-      offset += bind_offset.z * depth_pitch +
-                (bind_offset.y * pitch * surface->prt_tile_depth +
-                 bind_offset.x * surface->prt_tile_height * surface->prt_tile_depth) *
-                   bs;
-
-      uint32_t aligned_extent_width = ALIGN(bind_extent.width, surface->prt_tile_width);
-      uint32_t aligned_extent_height = ALIGN(bind_extent.height, surface->prt_tile_height);
-      uint32_t aligned_extent_depth = ALIGN(bind_extent.depth, surface->prt_tile_depth);
-
-      bool whole_subres =
-         (bind_extent.height <= surface->prt_tile_height || aligned_extent_width == pitch) &&
-         (bind_extent.depth <= surface->prt_tile_depth ||
-          aligned_extent_width * aligned_extent_height * bs == depth_pitch);
-
-      if (whole_subres) {
-         uint32_t size = aligned_extent_width * aligned_extent_height * aligned_extent_depth * bs;
-         result = device->ws->buffer_virtual_bind(device->ws, image->bindings[0].bo, offset, size,
-                                                  mem ? mem->bo : NULL, mem_offset);
-         if (result != VK_SUCCESS)
-            return result;
-
-         if (bind->pBinds[i].memory)
-            radv_rmv_log_sparse_add_residency(device, image->bindings[0].bo, offset);
-         else
-            radv_rmv_log_sparse_remove_residency(device, image->bindings[0].bo, offset);
-
-      } else {
-         uint32_t img_y_increment = pitch * bs * surface->prt_tile_depth;
-         uint32_t mem_y_increment = aligned_extent_width * bs * surface->prt_tile_depth;
-         uint32_t mem_z_increment = aligned_extent_width * aligned_extent_height * bs;
-         uint32_t size = mem_y_increment * surface->prt_tile_height;
-         for (unsigned z = 0; z < bind_extent.depth;
-              z += surface->prt_tile_depth, offset += depth_pitch * surface->prt_tile_depth) {
-            for (unsigned y = 0; y < bind_extent.height; y += surface->prt_tile_height) {
-               result = device->ws->buffer_virtual_bind(
-                  device->ws, image->bindings[0].bo, offset + img_y_increment * y, size,
-                  mem ? mem->bo : NULL, mem_offset + mem_y_increment * y + mem_z_increment * z);
-               if (result != VK_SUCCESS)
-                  return result;
-
-               if (bind->pBinds[i].memory)
-                  radv_rmv_log_sparse_add_residency(device, image->bindings[0].bo, offset);
-               else
-                  radv_rmv_log_sparse_remove_residency(device, image->bindings[0].bo, offset);
-            }
-         }
-      }
-   }
-
-   return VK_SUCCESS;
-}
-
-static VkResult
-radv_update_preambles(struct radv_queue_state *queue, struct radv_device *device,
-                      struct vk_command_buffer *const *cmd_buffers, uint32_t cmd_buffer_count,
-                      bool *use_perf_counters, bool *use_ace)
-{
-   if (queue->qf != RADV_QUEUE_GENERAL &&
-       queue->qf != RADV_QUEUE_COMPUTE)
-      return VK_SUCCESS;
-
-   /* Figure out the needs of the current submission.
-    * Start by copying the queue's current info.
-    * This is done because we only allow two possible behaviours for these buffers:
-    * - Grow when the newly needed amount is larger than what we had
-    * - Allocate the max size and reuse it, but don't free it until the queue is destroyed
-    */
-   struct radv_queue_ring_info needs = queue->ring_info;
-   *use_perf_counters = false;
-   *use_ace = false;
-
-   for (uint32_t j = 0; j < cmd_buffer_count; j++) {
-      struct radv_cmd_buffer *cmd_buffer = container_of(cmd_buffers[j], struct radv_cmd_buffer, vk);
-
-      needs.scratch_size_per_wave =
-         MAX2(needs.scratch_size_per_wave, cmd_buffer->scratch_size_per_wave_needed);
-      needs.scratch_waves = MAX2(needs.scratch_waves, cmd_buffer->scratch_waves_wanted);
-      needs.compute_scratch_size_per_wave = MAX2(needs.compute_scratch_size_per_wave,
-                                                 cmd_buffer->compute_scratch_size_per_wave_needed);
-      needs.compute_scratch_waves =
-         MAX2(needs.compute_scratch_waves, cmd_buffer->compute_scratch_waves_wanted);
-      needs.esgs_ring_size = MAX2(needs.esgs_ring_size, cmd_buffer->esgs_ring_size_needed);
-      needs.gsvs_ring_size = MAX2(needs.gsvs_ring_size, cmd_buffer->gsvs_ring_size_needed);
-      needs.tess_rings |= cmd_buffer->tess_rings_needed;
-      needs.task_rings |= cmd_buffer->task_rings_needed;
-      needs.mesh_scratch_ring |= cmd_buffer->mesh_scratch_ring_needed;
-      needs.gds |= cmd_buffer->gds_needed;
-      needs.gds_oa |= cmd_buffer->gds_oa_needed;
-      needs.sample_positions |= cmd_buffer->sample_positions_needed;
-      *use_perf_counters |= cmd_buffer->state.uses_perf_counters;
-      *use_ace |= !!cmd_buffer->ace_internal.cs;
-   }
-
-   /* Sanitize scratch size information. */
-   needs.scratch_waves = needs.scratch_size_per_wave
-                            ? MIN2(needs.scratch_waves, UINT32_MAX / needs.scratch_size_per_wave)
-                            : 0;
-   needs.compute_scratch_waves =
-      needs.compute_scratch_size_per_wave
-         ? MIN2(needs.compute_scratch_waves, UINT32_MAX / needs.compute_scratch_size_per_wave)
-         : 0;
-
-   if (device->physical_device->rad_info.gfx_level >= GFX11 && queue->qf == RADV_QUEUE_GENERAL) {
-      needs.attr_ring_size = device->physical_device->rad_info.attribute_ring_size_per_se *
-                             device->physical_device->rad_info.max_se;
-   }
-
-   /* Return early if we already match these needs.
-    * Note that it's not possible for any of the needed values to be less
-    * than what the queue already had, because we only ever increase the allocated size.
-    */
-   if (queue->initial_full_flush_preamble_cs &&
-       queue->ring_info.scratch_size_per_wave == needs.scratch_size_per_wave &&
-       queue->ring_info.scratch_waves == needs.scratch_waves &&
-       queue->ring_info.compute_scratch_size_per_wave == needs.compute_scratch_size_per_wave &&
-       queue->ring_info.compute_scratch_waves == needs.compute_scratch_waves &&
-       queue->ring_info.esgs_ring_size == needs.esgs_ring_size &&
-       queue->ring_info.gsvs_ring_size == needs.gsvs_ring_size &&
-       queue->ring_info.tess_rings == needs.tess_rings &&
-       queue->ring_info.task_rings == needs.task_rings &&
-       queue->ring_info.mesh_scratch_ring == needs.mesh_scratch_ring &&
-       queue->ring_info.attr_ring_size == needs.attr_ring_size &&
-       queue->ring_info.gds == needs.gds &&
-       queue->ring_info.gds_oa == needs.gds_oa &&
-       queue->ring_info.sample_positions == needs.sample_positions)
-      return VK_SUCCESS;
-
-   return radv_update_preamble_cs(queue, device, &needs);
-}
-
-static VkResult
-radv_create_gang_wait_preambles_postambles(struct radv_queue *queue)
-{
-   if (queue->gang_sem_bo)
-      return VK_SUCCESS;
-
-   VkResult r = VK_SUCCESS;
-   struct radv_device *device = queue->device;
-   struct radeon_winsys *ws = device->ws;
-   const enum amd_ip_type leader_ip = radv_queue_family_to_ring(device->physical_device, queue->state.qf);
-   struct radeon_winsys_bo *gang_sem_bo = NULL;
-
-   /* Gang semaphores BO.
-    * DWORD 0: used in preambles, gang leader writes, gang members wait.
-    * DWORD 1: used in postambles, gang leader waits, gang members write.
-    */
-   r = ws->buffer_create(ws, 8, 4, RADEON_DOMAIN_VRAM,
-                         RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_ZERO_VRAM, RADV_BO_PRIORITY_SCRATCH,
-                         0, &gang_sem_bo);
-   if (r != VK_SUCCESS)
-      return r;
-
-   struct radeon_cmdbuf *leader_pre_cs = ws->cs_create(ws, leader_ip);
-   struct radeon_cmdbuf *leader_post_cs = ws->cs_create(ws, leader_ip);
-   struct radeon_cmdbuf *ace_pre_cs = ws->cs_create(ws, AMD_IP_COMPUTE);
-   struct radeon_cmdbuf *ace_post_cs = ws->cs_create(ws, AMD_IP_COMPUTE);
-
-   if (!leader_pre_cs || !leader_post_cs || !ace_pre_cs || !ace_post_cs)
-      goto fail;
-
-   radv_cs_add_buffer(ws, leader_pre_cs, gang_sem_bo);
-   radv_cs_add_buffer(ws, leader_post_cs, gang_sem_bo);
-   radv_cs_add_buffer(ws, ace_pre_cs, gang_sem_bo);
-   radv_cs_add_buffer(ws, ace_post_cs, gang_sem_bo);
-
-   const uint64_t ace_wait_va = radv_buffer_get_va(gang_sem_bo);
-   const uint64_t leader_wait_va = ace_wait_va + 4;
-
-   /* Preambles for gang submission.
-    * Make gang members wait until the gang leader starts.
-    * Userspace is required to emit this wait to make sure it behaves correctly
-    * in a multi-process environment, because task shader dispatches are not
-    * meant to be executed on multiple compute engines at the same time.
-    */
-   radv_cp_wait_mem(ace_pre_cs, WAIT_REG_MEM_GREATER_OR_EQUAL, ace_wait_va, 1, 0xffffffff);
-   radeon_emit(ace_pre_cs, PKT3(PKT3_WRITE_DATA, 3, 0));
-   radeon_emit(ace_pre_cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME));
-   radeon_emit(ace_pre_cs, ace_wait_va);
-   radeon_emit(ace_pre_cs, ace_wait_va >> 32);
-   radeon_emit(ace_pre_cs, 0);
-   radeon_emit(leader_pre_cs, PKT3(PKT3_WRITE_DATA, 3, 0));
-   radeon_emit(leader_pre_cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME));
-   radeon_emit(leader_pre_cs, ace_wait_va);
-   radeon_emit(leader_pre_cs, ace_wait_va >> 32);
-   radeon_emit(leader_pre_cs, 1);
-
-   /* Create postambles for gang submission.
-    * This ensures that the gang leader waits for the whole gang,
-    * which is necessary because the kernel signals the userspace fence
-    * as soon as the gang leader is done, which may lead to bugs because the
-    * same command buffers could be submitted again while still being executed.
-    */
-   radv_cp_wait_mem(leader_post_cs, WAIT_REG_MEM_GREATER_OR_EQUAL, leader_wait_va, 1, 0xffffffff);
-   radeon_emit(leader_post_cs, PKT3(PKT3_WRITE_DATA, 3, 0));
-   radeon_emit(leader_post_cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME));
-   radeon_emit(leader_post_cs, leader_wait_va);
-   radeon_emit(leader_post_cs, leader_wait_va >> 32);
-   radeon_emit(leader_post_cs, 0);
-   radeon_emit(ace_post_cs, PKT3(PKT3_WRITE_DATA, 3, 0));
-   radeon_emit(ace_post_cs, S_370_DST_SEL(V_370_MEM) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(V_370_ME));
-   radeon_emit(ace_post_cs, leader_wait_va);
-   radeon_emit(ace_post_cs, leader_wait_va >> 32);
-   radeon_emit(ace_post_cs, 1);
-
-   r = ws->cs_finalize(leader_pre_cs);
-   if (r != VK_SUCCESS)
-      goto fail;
-   r = ws->cs_finalize(leader_post_cs);
-   if (r != VK_SUCCESS)
-      goto fail;
-   r = ws->cs_finalize(ace_pre_cs);
-   if (r != VK_SUCCESS)
-      goto fail;
-   r = ws->cs_finalize(ace_post_cs);
-   if (r != VK_SUCCESS)
-      goto fail;
-
-   queue->gang_sem_bo = gang_sem_bo;
-   queue->state.gang_wait_preamble_cs = leader_pre_cs;
-   queue->state.gang_wait_postamble_cs = leader_post_cs;
-   queue->ace_internal_state->gang_wait_preamble_cs = ace_pre_cs;
-   queue->ace_internal_state->gang_wait_postamble_cs = ace_post_cs;
-
-   return VK_SUCCESS;
-
-fail:
-   if (leader_pre_cs)
-      ws->cs_destroy(leader_pre_cs);
-   if (leader_post_cs)
-      ws->cs_destroy(leader_post_cs);
-   if (ace_pre_cs)
-      ws->cs_destroy(ace_pre_cs);
-   if (ace_post_cs)
-      ws->cs_destroy(ace_post_cs);
-   if (gang_sem_bo)
-      ws->buffer_destroy(ws, gang_sem_bo);
-
-   return r;
-}
-
-static VkResult
-radv_update_gang_preambles(struct radv_queue *queue)
-{
-   if (!radv_queue_init_ace_internal_state(queue))
-      return VK_ERROR_OUT_OF_HOST_MEMORY;
-
-   VkResult r = VK_SUCCESS;
-
-   /* Copy task rings state.
-    * Task shaders that are submitted on the ACE queue need to share
-    * their ring buffers with the mesh shaders on the GFX queue.
-    */
-   queue->ace_internal_state->ring_info.task_rings = queue->state.ring_info.task_rings;
-   queue->ace_internal_state->task_rings_bo = queue->state.task_rings_bo;
-
-   /* Copy some needed states from the parent queue state.
-    * These can only increase so it's okay to copy them as-is without checking.
-    * Note, task shaders use the scratch size from their graphics pipeline.
-    */
-   struct radv_queue_ring_info needs = queue->ace_internal_state->ring_info;
-   needs.compute_scratch_size_per_wave = queue->state.ring_info.scratch_size_per_wave;
-   needs.compute_scratch_waves = queue->state.ring_info.scratch_waves;
-   needs.task_rings = queue->state.ring_info.task_rings;
-
-   r = radv_update_preamble_cs(queue->ace_internal_state, queue->device, &needs);
-   if (r != VK_SUCCESS)
-      return r;
-
-   r = radv_create_gang_wait_preambles_postambles(queue);
-   if (r != VK_SUCCESS)
-      return r;
-
-   return VK_SUCCESS;
-}
-
-static bool
-radv_cmd_buffer_needs_ace(const struct radv_cmd_buffer *cmd_buffer)
-{
-   return cmd_buffer->ace_internal.cs && cmd_buffer->task_rings_needed;
-}
-
-static VkResult
-radv_queue_submit_bind_sparse_memory(struct radv_device *device, struct vk_queue_submit *submission)
-{
-   for (uint32_t i = 0; i < submission->buffer_bind_count; ++i) {
-      VkResult result = radv_sparse_buffer_bind_memory(device, submission->buffer_binds + i);
-      if (result != VK_SUCCESS)
-         return result;
-   }
-
-   for (uint32_t i = 0; i < submission->image_opaque_bind_count; ++i) {
-      VkResult result =
-         radv_sparse_image_opaque_bind_memory(device, submission->image_opaque_binds + i);
-      if (result != VK_SUCCESS)
-         return result;
-   }
-
-   for (uint32_t i = 0; i < submission->image_bind_count; ++i) {
-      VkResult result = radv_sparse_image_bind_memory(device, submission->image_binds + i);
-      if (result != VK_SUCCESS)
-         return result;
-   }
-
-   return VK_SUCCESS;
-}
-
-static VkResult
-radv_queue_submit_empty(struct radv_queue *queue, struct vk_queue_submit *submission)
-{
-   struct radeon_winsys_ctx *ctx = queue->hw_ctx;
-   struct radv_winsys_submit_info submit = {
-      .ip_type = radv_queue_ring(queue),
-      .queue_index = queue->vk.index_in_family,
-   };
-
-   return queue->device->ws->cs_submit(ctx, &submit, submission->wait_count, submission->waits,
-                                       submission->signal_count, submission->signals, false);
-}
-
-static VkResult
-radv_queue_submit_normal(struct radv_queue *queue, struct vk_queue_submit *submission)
-{
-   struct radeon_winsys_ctx *ctx = queue->hw_ctx;
-   bool can_patch = true;
-   bool use_ace = false;
-   bool use_perf_counters = false;
-   VkResult result;
-
-   result = radv_update_preambles(&queue->state, queue->device, submission->command_buffers,
-                                  submission->command_buffer_count, &use_perf_counters, &use_ace);
-   if (result != VK_SUCCESS)
-      return result;
-
-   if (use_ace) {
-      /* TODO: chaining with gang submit. */
-      can_patch = false;
-
-      result = radv_update_gang_preambles(queue);
-      if (result != VK_SUCCESS)
-         return result;
-   }
-
-   const unsigned num_perfctr_cs = use_perf_counters ? 2 : 0;
-   const unsigned num_gang_wait_cs = use_ace ? 4 : 0;
-   const unsigned max_cs_submission = queue->device->trace_bo ? 1 : RADV_MAX_IBS_PER_SUBMIT;
-   const unsigned cmd_buffer_count = submission->command_buffer_count;
-   const unsigned cs_array_size =
-      (use_ace ? 2 : 1) * MIN2(max_cs_submission, cmd_buffer_count) + num_perfctr_cs + num_gang_wait_cs;
-
-   struct radeon_cmdbuf **cs_array = malloc(sizeof(struct radeon_cmdbuf *) * cs_array_size);
-   if (!cs_array)
-      return VK_ERROR_OUT_OF_HOST_MEMORY;
-
-   if (queue->device->trace_bo)
-      simple_mtx_lock(&queue->device->trace_mtx);
-
-   struct radeon_cmdbuf *perf_ctr_lock_cs = NULL;
-   struct radeon_cmdbuf *perf_ctr_unlock_cs = NULL;
-
-   if (use_perf_counters) {
-      /* Create the lock/unlock CS. */
-      perf_ctr_lock_cs =
-         radv_create_perf_counter_lock_cs(queue->device, submission->perf_pass_index, false);
-      perf_ctr_unlock_cs =
-         radv_create_perf_counter_lock_cs(queue->device, submission->perf_pass_index, true);
-
-      /* RADV only supports perf counters on the GFX queue currently. */
-      assert(radv_queue_ring(queue) == AMD_IP_GFX);
-
-      /* Disallow chaining the lock/unlock CS. */
-      can_patch = false;
-
-      if (!perf_ctr_lock_cs || !perf_ctr_unlock_cs) {
-         result = VK_ERROR_OUT_OF_HOST_MEMORY;
-         goto fail;
-      }
-   }
-
-   /* For fences on the same queue/vm amdgpu doesn't wait till all processing is finished
-    * before starting the next cmdbuffer, so we need to do it here.
-    */
-   const bool need_wait = submission->wait_count > 0;
-   unsigned num_preambles = 0;
-   struct radeon_cmdbuf *preambles[4] = {0};
-
-   if (queue->state.qf == RADV_QUEUE_GENERAL || queue->state.qf == RADV_QUEUE_COMPUTE) {
-      preambles[num_preambles++] =
-         need_wait ? queue->state.initial_full_flush_preamble_cs : queue->state.initial_preamble_cs;
-   }
-
-   const unsigned num_1q_preambles = num_preambles;
-
-   if (use_ace) {
-      preambles[num_preambles++] = queue->state.gang_wait_preamble_cs;
-      preambles[num_preambles++] = queue->ace_internal_state->gang_wait_preamble_cs;
-      preambles[num_preambles++] = need_wait
-                                      ? queue->ace_internal_state->initial_full_flush_preamble_cs
-                                      : queue->ace_internal_state->initial_preamble_cs;
-   }
-
-   struct radv_winsys_submit_info submit = {
-      .ip_type = radv_queue_ring(queue),
-      .queue_index = queue->vk.index_in_family,
-      .cs_array = cs_array,
-      .cs_count = 0,
-      .preamble_count = 1,
-      .initial_preamble_cs = preambles,
-      .continue_preamble_cs = queue->state.continue_preamble_cs,
-      .uses_shadow_regs = queue->state.uses_shadow_regs,
-   };
-
-   for (uint32_t j = 0, advance; j < cmd_buffer_count; j += advance) {
-      advance = MIN2(max_cs_submission, cmd_buffer_count - j);
-      const bool last_submit = j + advance == cmd_buffer_count;
-      bool submit_ace = false;
-      unsigned num_submitted_cs = 0;
-      unsigned cs_idx = 0;
-
-      if (queue->device->trace_bo)
-         *queue->device->trace_id_ptr = 0;
-
-      /* Add perf counter lock CS (GFX only) to the beginning of each submit. */
-      if (use_perf_counters)
-         cs_array[num_submitted_cs++] = perf_ctr_lock_cs;
-
-      /* Add CS from submitted command buffers. */
-      for (unsigned c = 0; c < advance; ++c) {
-         struct radv_cmd_buffer *cmd_buffer =
-            (struct radv_cmd_buffer *)submission->command_buffers[j + c];
-         assert(cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
-
-         /* ACE needs to be first because the last CS must match the queue's IP type. */
-         if (radv_cmd_buffer_needs_ace(cmd_buffer)) {
-            cs_array[num_submitted_cs++] = cmd_buffer->ace_internal.cs;
-            submit_ace = true;
-         }
-
-         can_patch &= !(cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT);
-         cs_array[num_submitted_cs++] = cmd_buffer->cs;
-         cs_idx = num_submitted_cs - 1;
-      }
-
-      /* Add gang wait postambles to make sure the gang leader waits for the whole gang. */
-      if (submit_ace) {
-         cs_array[num_submitted_cs++] = queue->ace_internal_state->gang_wait_postamble_cs;
-         cs_array[num_submitted_cs++] = queue->state.gang_wait_postamble_cs;
-      }
-
-      /* Add perf counter unlock CS (GFX only) to the end of each submit. */
-      if (use_perf_counters)
-         cs_array[num_submitted_cs++] = perf_ctr_unlock_cs;
-
-      submit.cs_count = num_submitted_cs;
-      submit.preamble_count = submit_ace ? num_preambles : num_1q_preambles;
-
-      result = queue->device->ws->cs_submit(
-         ctx, &submit, j == 0 ? submission->wait_count : 0, submission->waits,
-         last_submit ? submission->signal_count : 0, submission->signals, can_patch);
-
-      if (result != VK_SUCCESS)
-         goto fail;
-
-      if (queue->device->trace_bo) {
-         radv_check_gpu_hangs(queue, cs_array[cs_idx]);
-      }
-
-      if (queue->device->tma_bo) {
-         radv_check_trap_handler(queue);
-      }
-
-      preambles[0] = queue->state.initial_preamble_cs;
-      preambles[1] = !use_ace ? NULL : queue->ace_internal_state->initial_preamble_cs;
-   }
-
-fail:
-   free(cs_array);
-   if (queue->device->trace_bo)
-      simple_mtx_unlock(&queue->device->trace_mtx);
-
-   return result;
-}
-
-static VkResult
-radv_queue_submit(struct vk_queue *vqueue, struct vk_queue_submit *submission)
-{
-   struct radv_queue *queue = (struct radv_queue *)vqueue;
-   VkResult result;
-
-   radv_rmv_log_submit(queue->device, radv_queue_ring(queue));
-
-   result = radv_queue_submit_bind_sparse_memory(queue->device, submission);
-   if (result != VK_SUCCESS)
-      goto fail;
-
-   if (!submission->command_buffer_count && !submission->wait_count && !submission->signal_count)
-      return VK_SUCCESS;
-
-   if (!submission->command_buffer_count) {
-      result = radv_queue_submit_empty(queue, submission);
-   } else {
-      result = radv_queue_submit_normal(queue, submission);
-   }
-
-fail:
-   if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) {
-      /* When something bad happened during the submission, such as
-       * an out of memory issue, it might be hard to recover from
-       * this inconsistent state. To avoid this sort of problem, we
-       * assume that we are in a really bad situation and return
-       * VK_ERROR_DEVICE_LOST to ensure the clients do not attempt
-       * to submit the same job again to this device.
-       */
-      result = vk_device_set_lost(&queue->device->vk, "vkQueueSubmit() failed");
-   }
-   return result;
-}
-
-bool
-radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs)
-{
-   struct radeon_winsys_ctx *ctx = queue->hw_ctx;
-   struct radv_winsys_submit_info submit = {
-      .ip_type = radv_queue_ring(queue),
-      .queue_index = queue->vk.index_in_family,
-      .cs_array = &cs,
-      .cs_count = 1,
-   };
-
-   VkResult result = queue->device->ws->cs_submit(ctx, &submit, 0, NULL, 0, NULL, false);
-   if (result != VK_SUCCESS)
-      return false;
-
-   return true;
-}
-
 bool
 radv_get_memory_fd(struct radv_device *device, struct radv_device_memory *memory, int *pFD)
 {