virgl/ci: Run each dEQP instance in its own VM

Currently we run deqp-runner inside a single VM, which makes very poor
use of the available CPUs because Virgl has a bottleneck in the VMM that
serializes everything.

With this change, we can run several Crosvm instances in a runner and
make full use of the CPUs. Getting the same coverage with 3 runners
instead of 6.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Emma Anholt <emma@anholt.net>
Reviewed-by: Corentin Noël <corentin.noel@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12828>

10 files changed, 102 insertions, 145 deletions

diff --git a/src/gallium/drivers/virgl/ci/deqp-virgl-gl.toml b/src/gallium/drivers/virgl/ci/deqp-virgl-gl.toml
index f65be42cc18..fc874c49fb9 100644
--- a/src/gallium/drivers/virgl/ci/deqp-virgl-gl.toml
+++ b/src/gallium/drivers/virgl/ci/deqp-virgl-gl.toml
@@ -1,51 +1,57 @@
 [[deqp]]
-deqp = "/deqp/modules/gles2/deqp-gles2"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles2-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles2/deqp-gles2",
     "--deqp-surface-width=256",
     "--deqp-surface-height=256",
     "--deqp-surface-type=pbuffer",
     "--deqp-gl-config-name=rgba8888d24s8ms0",
     "--deqp-visibility=hidden"
 ]
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
 version_check = "GL ES 3.2.*git"
-renderer_check = "virgl"
+renderer_check = "virgl.*llvmpipe"
 
 [[deqp]]
-deqp = "/deqp/modules/gles3/deqp-gles3"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles3-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles3/deqp-gles3",
     "--deqp-surface-width=256",
     "--deqp-surface-height=256",
     "--deqp-surface-type=pbuffer",
     "--deqp-gl-config-name=rgba8888d24s8ms0",
     "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
 
 [[deqp]]
-deqp = "/deqp/modules/gles31/deqp-gles31"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles31-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles31/deqp-gles31",
     "--deqp-surface-width=256",
     "--deqp-surface-height=256",
     "--deqp-surface-type=pbuffer",
     "--deqp-gl-config-name=rgba8888d24s8ms0",
     "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
 
 [[deqp]]
-deqp = "/deqp/external/openglcts/modules/glcts"
+deqp = "/install/crosvm-runner.sh"
 caselists = [
     "/deqp/mustpass/gl30-master.txt",
     "/deqp/mustpass/gl31-master.txt",
     "/deqp/mustpass/gl32-master.txt",
 ]
 deqp_args = [
+    "/deqp/external/openglcts/modules/glcts",
     "--deqp-surface-width=256",
     "--deqp-surface-height=256",
     "--deqp-surface-type=pbuffer",
     "--deqp-gl-config-name=rgba8888d24s8ms0",
     "--deqp-visibility=hidden"
 ]
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
diff --git a/src/gallium/drivers/virgl/ci/deqp-virgl-gles.toml b/src/gallium/drivers/virgl/ci/deqp-virgl-gles.toml
index a332f152ba3..fc874c49fb9 100644
--- a/src/gallium/drivers/virgl/ci/deqp-virgl-gles.toml
+++ b/src/gallium/drivers/virgl/ci/deqp-virgl-gles.toml
@@ -1,36 +1,57 @@
 [[deqp]]
-deqp = "/deqp/modules/gles2/deqp-gles2"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles2-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles2/deqp-gles2",
     "--deqp-surface-width=256",
     "--deqp-surface-height=256",
     "--deqp-surface-type=pbuffer",
     "--deqp-gl-config-name=rgba8888d24s8ms0",
     "--deqp-visibility=hidden"
 ]
-version_check = "GL ES 3.1.*git"
-renderer_check = "virgl"
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
+version_check = "GL ES 3.2.*git"
+renderer_check = "virgl.*llvmpipe"
 
 [[deqp]]
-deqp = "/deqp/modules/gles3/deqp-gles3"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles3-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles3/deqp-gles3",
     "--deqp-surface-width=256",
     "--deqp-surface-height=256",
     "--deqp-surface-type=pbuffer",
     "--deqp-gl-config-name=rgba8888d24s8ms0",
     "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
 
 [[deqp]]
-deqp = "/deqp/modules/gles31/deqp-gles31"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles31-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles31/deqp-gles31",
     "--deqp-surface-width=256",
     "--deqp-surface-height=256",
     "--deqp-surface-type=pbuffer",
     "--deqp-gl-config-name=rgba8888d24s8ms0",
     "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
+
+[[deqp]]
+deqp = "/install/crosvm-runner.sh"
+caselists = [
+    "/deqp/mustpass/gl30-master.txt",
+    "/deqp/mustpass/gl31-master.txt",
+    "/deqp/mustpass/gl32-master.txt",
+]
+deqp_args = [
+    "/deqp/external/openglcts/modules/glcts",
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
diff --git a/src/gallium/drivers/virgl/ci/gitlab-ci.yml b/src/gallium/drivers/virgl/ci/gitlab-ci.yml
index f046e0c5311..43aa62a2496 100644
--- a/src/gallium/drivers/virgl/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/virgl/ci/gitlab-ci.yml
@@ -5,22 +5,17 @@
   variables:
     GALLIUM_DRIVER: "virgl"
     CROSVM_GALLIUM_DRIVER: "llvmpipe"
-    CROSVM_GPU_ARGS: "gles=false,backend=virglrenderer,egl=true,surfaceless=true"
+    CROSVM_GPU_ARGS: "gles=false,backend=virglrenderer,egl=true,surfaceless=true,width=1024,height=768"
     GALLIVM_PERF: "nopt,no_quad_lod"
 
 virgl-on-gl:
   variables:
     DEQP_SUITE: virgl-gl
     GPU_VERSION: virgl-gl
-    # Speed rendering a bit, as crosvm is processing requests serially
-    LP_NUM_THREADS: 8
-    FDO_CI_CONCURRENT: 3
-    CROSVM_TEST_SCRIPT: "/install/deqp-runner.sh"
-  parallel: 6
+    LP_NUM_THREADS: 1 # There will be FDO_CI_CONCURRENT Crosvm processes, so each should use a single thread
+  parallel: 3
   tags:
     - kvm
-  script:
-    - install/crosvm-runner.sh
   extends:
     - .deqp-test
     - .virgl-test
@@ -30,10 +25,9 @@ virgl-on-gles:
     VIRGL_HOST_API: GLES
     DEQP_SUITE: virgl-gles
     GPU_VERSION: virgl-gles
-    CROSVM_GPU_ARGS: "gles=true,backend=virglrenderer,egl=true,surfaceless=true"
+    CROSVM_GPU_ARGS: "gles=true,backend=virglrenderer,egl=true,surfaceless=true,width=1024,height=768"
   extends:
     - virgl-on-gl
-    - .test-manual
 
 virgl-traces:
   extends:
@@ -44,9 +38,10 @@ virgl-traces:
     PIGLIT_REPLAY_DESCRIPTION_FILE: "${CI_PROJECT_DIR}/install/traces-virgl.yml"
     PIGLIT_REPLAY_DEVICE_NAME: "gl-virgl"
     PIGLIT_RESULTS: "virgl-replay"
-    CROSVM_TEST_SCRIPT: "/install/piglit/run.sh"
+    LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/install/lib/"     # For Crosvm
   tags:
     - kvm
   script:
-    - install/crosvm-runner.sh
-
+    # Use all threads for rendering and only run one job at a time
+    # Couldn't get GitLab CI to correctly substitute the variable in the yaml
+    - LP_NUM_THREADS=${FDO_CI_CONCURRENT} FDO_CI_CONCURRENT=1 install/crosvm-runner.sh install/piglit/run.sh
diff --git a/src/gallium/drivers/virgl/ci/traces-virgl.yml b/src/gallium/drivers/virgl/ci/traces-virgl.yml
index b0cbf98b568..53a35057235 100644
--- a/src/gallium/drivers/virgl/ci/traces-virgl.yml
+++ b/src/gallium/drivers/virgl/ci/traces-virgl.yml
@@ -231,7 +231,7 @@ traces:
   - path: supertuxkart/supertuxkart-mansion-egl-gles.trace
     expectations:
       - device: gl-virgl
-        checksum: 156c26de2cefe1973b1593e6b22f7edb
+        checksum: c8e5d7c4377b8e484ae41270692914f3
   - path: xonotic/xonotic-keybench-high.trace
     expectations:
       - device: gl-virgl
diff --git a/src/gallium/drivers/virgl/ci/virgl-gl-fails.txt b/src/gallium/drivers/virgl/ci/virgl-gl-fails.txt
index 794e0ce8aa4..9f27f03fb41 100644
--- a/src/gallium/drivers/virgl/ci/virgl-gl-fails.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gl-fails.txt
@@ -1,8 +1,6 @@
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner,Fail
 dEQP-GLES2.functional.clipping.point.wide_point_clip,Fail
-dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center,Fail
-dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_corner,Fail
 dEQP-GLES3.functional.clipping.point.wide_point_clip,Fail
@@ -20,8 +18,7 @@ dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_dst_y,Fa
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_x,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.4,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_implementation_draw_buffers.8,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_12,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_writeonly_12,Fail
+dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.9,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_equal,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_larger,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.fbo_bbox_equal,Fail
@@ -40,6 +37,7 @@ dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_rbo_2,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_1,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_2,Fail
+dEQP-GLES31.functional.ssbo.layout.random.all_shared_buffer.36,Fail
 # https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2892
 KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x2_vs,Crash
 KHR-GL30.transform_feedback.api_errors_test,Fail
diff --git a/src/gallium/drivers/virgl/ci/virgl-gl-flakes.txt b/src/gallium/drivers/virgl/ci/virgl-gl-flakes.txt
index cc7c6b401b9..d5c22fa62cc 100644
--- a/src/gallium/drivers/virgl/ci/virgl-gl-flakes.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gl-flakes.txt
@@ -4,3 +4,14 @@ KHR-GL30.shaders30.glsl_constructors.bvec4_from_bool_mat3_vs
 KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x3_vs
 # https://gitlab.freedesktop.org/mesa/mesa/-/issues/4651
 KHR-GL31.transform_feedback.capture_special_interleaved_test
+
+dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_12
+dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_12
+
+dEQP-GLES2.functional.clipping.point.wide_point_clip
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner
+
+dEQP-GLES3.functional.clipping.point.wide_point_clip
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_corner
diff --git a/src/gallium/drivers/virgl/ci/virgl-gl-skips.txt b/src/gallium/drivers/virgl/ci/virgl-gl-skips.txt
index 415a8813670..4061d4ef47c 100644
--- a/src/gallium/drivers/virgl/ci/virgl-gl-skips.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gl-skips.txt
@@ -1,2 +1,6 @@
+# Note: skips lists for CI are just a list of lines that, when
+# non-zero-length and not starting with '#', will regex match to
+# delete lines from the test list.  Be careful.
+
 # Sometimes crashes, e.g. https://gitlab.freedesktop.org/kusma/mesa/-/jobs/4109419
 dEQP-GLES31.functional.compute.basic.empty
diff --git a/src/gallium/drivers/virgl/ci/virgl-gles-fails.txt b/src/gallium/drivers/virgl/ci/virgl-gles-fails.txt
index 29db955239d..9e92815ae63 100644
--- a/src/gallium/drivers/virgl/ci/virgl-gles-fails.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gles-fails.txt
@@ -1,13 +1,10 @@
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner,Fail
-dEQP-GLES2.functional.clipping.point.wide_point_clip,Fail
 dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center,Fail
 dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner,Fail
-dEQP-GLES2.functional.polygon_offset.default_displacement_with_units,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_corner,Fail
 dEQP-GLES3.functional.clipping.point.wide_point_clip,Fail
-dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_center,Fail
 dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_corner,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_mag,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_mag_reverse_dst_x,Fail
@@ -19,13 +16,9 @@ dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_dst_x,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_dst_x,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_dst_y,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_x,Fail
-dEQP-GLES3.functional.polygon_offset.default_displacement_with_units,Fail
-dEQP-GLES3.functional.polygon_offset.fixed24_displacement_with_units,Fail
-dEQP-GLES3.functional.polygon_offset.float32_displacement_with_units,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.4,Fail
+dEQP-GLES31.functional.draw_buffers_indexed.random.max_implementation_draw_buffers.8,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.9,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_12,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_32,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_equal,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_larger,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.fbo_bbox_equal,Fail
@@ -36,110 +29,6 @@ dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_d
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_draw.vertex_tessellation_fragment.fbo_bbox_larger,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_primitive.vertex_tessellation_fragment.default_framebuffer,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_primitive.vertex_tessellation_fragment.fbo,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_atomic_counter_bindings,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_atomic_counter_buffer_size,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_shader_output_resources,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_uniform_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_work_group_count,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_work_group_size,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_draw_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_input_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_uniform_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_program_texel_offset,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_output_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_uniform_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.min_program_texel_offset,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_input_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_output_vertices,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_total_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_uniform_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.sample_variables.max_samples,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_patch_vertices,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_input_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_total_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_input_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_uniform_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_gen_level,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_patch_components,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.atomic_counter.const_expression_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.atomic_counter.dynamically_uniform_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.atomic_counter.uniform_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.sampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usamplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isamplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.sampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.sampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.sampler2dshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.usamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ssbo.const_expression_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ssbo.const_literal_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ubo.const_expression_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ubo.dynamically_uniform_tessellation_control,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_pixel.multisample_rbo_1,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_pixel.multisample_rbo_2,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_pixel.multisample_texture_1,Fail
@@ -148,6 +37,11 @@ dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_rbo_2,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_1,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_2,Fail
+dEQP-GLES31.functional.ssbo.layout.random.all_shared_buffer.36,Fail
+dEQP-GLES31.functional.ssbo.layout.random.nested_structs_arrays_instance_arrays.22,Fail
+KHR-GL30.glsl_noperspective.functionaltest,Fail
+# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2892
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x2_vs,Crash
 KHR-GL30.transform_feedback.api_errors_test,Fail
 KHR-GL30.transform_feedback.capture_vertex_interleaved_test,Fail
 KHR-GL30.transform_feedback.capture_vertex_separate_test,Fail
@@ -158,13 +52,15 @@ KHR-GL30.transform_feedback.get_xfb_varying,Fail
 KHR-GL30.transform_feedback.query_vertex_interleaved_test,Fail
 KHR-GL30.transform_feedback.query_vertex_separate_test,Fail
 KHR-GL31.CommonBugs.CommonBug_ParenthesisInLayoutQualifierIntegerValue,Fail
+KHR-GL31.transform_feedback.capture_special_interleaved_test,Crash
 KHR-GL31.transform_feedback.capture_vertex_interleaved_test,Fail
 KHR-GL31.transform_feedback.capture_vertex_separate_test,Fail
 KHR-GL31.transform_feedback.discard_vertex_test,Fail
-KHR-GL31.transform_feedback.draw_xfb_instanced_test,Fail
+KHR-GL31.transform_feedback.draw_xfb_instanced_test,Crash
 KHR-GL31.transform_feedback.draw_xfb_stream_instanced_test,Crash
 KHR-GL31.transform_feedback.query_vertex_interleaved_test,Fail
 KHR-GL31.transform_feedback.query_vertex_separate_test,Fail
+KHR-GL31.glsl_noperspective.functionaltest,Fail
 KHR-GL32.CommonBugs.CommonBug_ParenthesisInLayoutQualifierIntegerValue,Fail
 KHR-GL32.transform_feedback.capture_vertex_interleaved_test,Fail
 KHR-GL32.transform_feedback.capture_vertex_separate_test,Fail
@@ -179,3 +75,4 @@ KHR-GL32.transform_feedback_overflow_query_ARB.multiple-streams-multiple-buffers
 KHR-GL32.transform_feedback_overflow_query_ARB.multiple-streams-one-buffer-per-stream,Fail
 KHR-GL32.transform_feedback.query_vertex_interleaved_test,Fail
 KHR-GL32.transform_feedback.query_vertex_separate_test,Fail
+KHR-GL32.glsl_noperspective.functionaltest,Fail
\ No newline at end of file
diff --git a/src/gallium/drivers/virgl/ci/virgl-gles-flakes.txt b/src/gallium/drivers/virgl/ci/virgl-gles-flakes.txt
new file mode 100644
index 00000000000..d5c22fa62cc
--- /dev/null
+++ b/src/gallium/drivers/virgl/ci/virgl-gles-flakes.txt
@@ -0,0 +1,17 @@
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x2_vs
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_bool_mat4x3_vs
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_bool_mat3_vs
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x3_vs
+# https://gitlab.freedesktop.org/mesa/mesa/-/issues/4651
+KHR-GL31.transform_feedback.capture_special_interleaved_test
+
+dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_12
+dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_12
+
+dEQP-GLES2.functional.clipping.point.wide_point_clip
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner
+
+dEQP-GLES3.functional.clipping.point.wide_point_clip
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_corner
diff --git a/src/gallium/drivers/virgl/ci/virgl-gles-skips.txt b/src/gallium/drivers/virgl/ci/virgl-gles-skips.txt
new file mode 100644
index 00000000000..d47f896e062
--- /dev/null
+++ b/src/gallium/drivers/virgl/ci/virgl-gles-skips.txt
@@ -0,0 +1,8 @@
+# Note: skips lists for CI are just a list of lines that, when
+# non-zero-length and not starting with '#', will regex match to
+# delete lines from the test list.  Be careful.
+
+KHR-GL32.packed_pixels.varied_rectangle.depth*
+
+# Sometimes crashes, e.g. https://gitlab.freedesktop.org/kusma/mesa/-/jobs/4109419
+dEQP-GLES31.functional.compute.basic.empty