venus: init exp features before ring init again

This reverts commit dda85cf94bdd9d58f0dc4013dbf4b3c985471991 ("venus:
move exp features init back to use ring submit"), and additionally adds
per stream shmem caching to determine when vkSetReplyCommandStreamMESA
is needed.

Checking renderer features before setting up ring means that the bound
shmem for replies on the ring will no longer be implicitly set on first
shmem creation (it was set for the renderer stream instead). So the
test for when another vkSetReplyCommandStreamMESA is needed must
independently consider the last stream set on renderer/ring(s)

Signed-off-by: Ryan Neph <ryanneph@google.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22036>

2 files changed, 78 insertions, 11 deletions

diff --git a/src/virtio/vulkan/vn_instance.c b/src/virtio/vulkan/vn_instance.c
index 2c6a4e8c6c5..7b5b475c401 100644
--- a/src/virtio/vulkan/vn_instance.c
+++ b/src/virtio/vulkan/vn_instance.c
@@ -163,6 +163,11 @@ vn_instance_init_ring(struct vn_instance *instance)
    return VK_SUCCESS;
 }
 
+static struct vn_renderer_shmem *
+vn_instance_get_reply_shmem_locked(struct vn_instance *instance,
+                                   size_t size,
+                                   void **ptr);
+
 static VkResult
 vn_instance_init_experimental_features(struct vn_instance *instance)
 {
@@ -174,8 +179,37 @@ vn_instance_init_experimental_features(struct vn_instance *instance)
    }
 
    size_t struct_size = sizeof(instance->experimental);
-   vn_call_vkGetVenusExperimentalFeatureData100000MESA(
-      instance, &struct_size, &instance->experimental);
+
+   /* prepare the reply shmem */
+   const size_t reply_size =
+      vn_sizeof_vkGetVenusExperimentalFeatureData100000MESA_reply(
+         &struct_size, &instance->experimental);
+   void *reply_ptr;
+   struct vn_renderer_shmem *reply_shmem =
+      vn_instance_get_reply_shmem_locked(instance, reply_size, &reply_ptr);
+   if (!reply_shmem)
+      return VK_ERROR_OUT_OF_HOST_MEMORY;
+
+   /* encode the command */
+   uint32_t local_data[16];
+   struct vn_cs_encoder local_enc =
+      VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
+   vn_encode_vkGetVenusExperimentalFeatureData100000MESA(
+      &local_enc, VK_COMMAND_GENERATE_REPLY_BIT_EXT, &struct_size,
+      &instance->experimental);
+
+   VkResult result = vn_renderer_submit_simple_sync(
+      instance->renderer, local_data, vn_cs_encoder_get_len(&local_enc));
+   if (result != VK_SUCCESS) {
+      vn_renderer_shmem_unref(instance->renderer, reply_shmem);
+      return result;
+   }
+
+   struct vn_cs_decoder reply_dec =
+      VN_CS_DECODER_INITIALIZER(reply_ptr, reply_size);
+   vn_decode_vkGetVenusExperimentalFeatureData100000MESA_reply(
+      &reply_dec, &struct_size, &instance->experimental);
+   vn_renderer_shmem_unref(instance->renderer, reply_shmem);
 
    VkVenusExperimentalFeatures100000MESA *exp_feats = &instance->experimental;
 
@@ -545,7 +579,6 @@ vn_instance_get_reply_shmem_locked(struct vn_instance *instance,
 {
    VN_TRACE_FUNC();
    struct vn_renderer_shmem_pool *pool = &instance->reply_shmem_pool;
-   const struct vn_renderer_shmem *saved_pool_shmem = pool->shmem;
 
    size_t offset;
    struct vn_renderer_shmem *shmem =
@@ -556,7 +589,14 @@ vn_instance_get_reply_shmem_locked(struct vn_instance *instance,
    assert(shmem == pool->shmem);
    *out_ptr = shmem->mmap_ptr + offset;
 
-   if (shmem != saved_pool_shmem) {
+   bool needs_set_stream = false;
+   if (likely(instance->ring.id)) {
+      needs_set_stream = shmem != instance->ring.reply_shmem;
+   } else {
+      needs_set_stream = shmem != instance->renderer_reply_shmem;
+   }
+
+   if (needs_set_stream) {
       uint32_t set_reply_command_stream_data[16];
       struct vn_cs_encoder local_enc = VN_CS_ENCODER_INITIALIZER_LOCAL(
          set_reply_command_stream_data,
@@ -567,7 +607,19 @@ vn_instance_get_reply_shmem_locked(struct vn_instance *instance,
       };
       vn_encode_vkSetReplyCommandStreamMESA(&local_enc, 0, &stream);
       vn_cs_encoder_commit(&local_enc);
-      vn_instance_ring_submit_locked(instance, &local_enc, NULL, NULL);
+
+      /* vn_instance_init_experimental_features calls this before the ring is
+       * created
+       */
+      if (likely(instance->ring.id)) {
+         vn_instance_ring_submit_locked(instance, &local_enc, NULL, NULL);
+         instance->ring.reply_shmem = shmem;
+      } else {
+         vn_renderer_submit_simple(instance->renderer,
+                                   set_reply_command_stream_data,
+                                   vn_cs_encoder_get_len(&local_enc));
+         instance->renderer_reply_shmem = shmem;
+      }
    }
 
    /* TODO avoid this seek command and go lock-free? */
@@ -576,7 +628,17 @@ vn_instance_get_reply_shmem_locked(struct vn_instance *instance,
       seek_reply_command_stream_data, sizeof(seek_reply_command_stream_data));
    vn_encode_vkSeekReplyCommandStreamMESA(&local_enc, 0, offset);
    vn_cs_encoder_commit(&local_enc);
-   vn_instance_ring_submit_locked(instance, &local_enc, NULL, NULL);
+
+   /* vn_instance_init_experimental_features calls this before the ring is
+    * created
+    */
+   if (likely(instance->ring.id)) {
+      vn_instance_ring_submit_locked(instance, &local_enc, NULL, NULL);
+   } else {
+      vn_renderer_submit_simple(instance->renderer,
+                                seek_reply_command_stream_data,
+                                vn_cs_encoder_get_len(&local_enc));
+   }
 
    return shmem;
 }
@@ -712,11 +774,11 @@ vn_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
    vn_renderer_shmem_pool_init(instance->renderer,
                                &instance->reply_shmem_pool, 1u << 20);
 
-   result = vn_instance_init_ring(instance);
+   result = vn_instance_init_experimental_features(instance);
    if (result != VK_SUCCESS)
       goto fail;
 
-   result = vn_instance_init_experimental_features(instance);
+   result = vn_instance_init_ring(instance);
    if (result != VK_SUCCESS)
       goto fail;
 
diff --git a/src/virtio/vulkan/vn_instance.h b/src/virtio/vulkan/vn_instance.h
index c4f3419cde1..86d2001957c 100644
--- a/src/virtio/vulkan/vn_instance.h
+++ b/src/virtio/vulkan/vn_instance.h
@@ -40,16 +40,24 @@ struct vn_instance {
    struct vn_renderer *renderer;
 
    struct vn_renderer_shmem_pool reply_shmem_pool;
+   /* cache "set" stream for renderer submission replies */
+   const struct vn_renderer_shmem *renderer_reply_shmem;
 
    mtx_t ring_idx_mutex;
    uint64_t ring_idx_used_mask;
 
+   /* XXX staged features to be merged to core venus protocol */
+   VkVenusExperimentalFeatures100000MESA experimental;
+
    struct {
       mtx_t mutex;
       struct vn_renderer_shmem *shmem;
       struct vn_ring ring;
       uint64_t id;
 
+      /* cache "set" stream for ring submission replies */
+      const struct vn_renderer_shmem *reply_shmem;
+
       struct vn_cs_encoder upload;
       uint32_t command_dropped;
 
@@ -58,9 +66,6 @@ struct vn_instance {
       uint64_t roundtrip_next;
    } ring;
 
-   /* XXX staged features to be merged to core venus protocol */
-   VkVenusExperimentalFeatures100000MESA experimental;
-
    /* Between the driver and the app, VN_MAX_API_VERSION is what we advertise
     * and base.base.app_info.api_version is what the app requests.
     *