radeon: fix stencil miptree allocation of combined ZS buffers on EG and SI

This allows texturing with depth-stencil buffers directly without the copy
to CB. The separate miptree description for stencil is added, because
the stencil mipmap offsets are not really depth offsets/4 (at least
for the texture units).

Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

2 files changed, 123 insertions, 76 deletions

diff --git a/radeon/radeon_surface.c b/radeon/radeon_surface.c
index c62cbf96..66c24449 100644
--- a/radeon/radeon_surface.c
+++ b/radeon/radeon_surface.c
@@ -144,31 +144,32 @@ static unsigned mip_minify(unsigned size, unsigned level)
 }
 
 static void surf_minify(struct radeon_surface *surf,
-                        unsigned level,
+                        struct radeon_surface_level *surflevel,
+                        unsigned bpe, unsigned level,
                         uint32_t xalign, uint32_t yalign, uint32_t zalign,
                         unsigned offset)
 {
-    surf->level[level].npix_x = mip_minify(surf->npix_x, level);
-    surf->level[level].npix_y = mip_minify(surf->npix_y, level);
-    surf->level[level].npix_z = mip_minify(surf->npix_z, level);
-    surf->level[level].nblk_x = (surf->level[level].npix_x + surf->blk_w - 1) / surf->blk_w;
-    surf->level[level].nblk_y = (surf->level[level].npix_y + surf->blk_h - 1) / surf->blk_h;
-    surf->level[level].nblk_z = (surf->level[level].npix_z + surf->blk_d - 1) / surf->blk_d;
-    if (surf->nsamples == 1 && surf->level[level].mode == RADEON_SURF_MODE_2D) {
-        if (surf->level[level].nblk_x < xalign || surf->level[level].nblk_y < yalign) {
-            surf->level[level].mode = RADEON_SURF_MODE_1D;
+    surflevel->npix_x = mip_minify(surf->npix_x, level);
+    surflevel->npix_y = mip_minify(surf->npix_y, level);
+    surflevel->npix_z = mip_minify(surf->npix_z, level);
+    surflevel->nblk_x = (surflevel->npix_x + surf->blk_w - 1) / surf->blk_w;
+    surflevel->nblk_y = (surflevel->npix_y + surf->blk_h - 1) / surf->blk_h;
+    surflevel->nblk_z = (surflevel->npix_z + surf->blk_d - 1) / surf->blk_d;
+    if (surf->nsamples == 1 && surflevel->mode == RADEON_SURF_MODE_2D) {
+        if (surflevel->nblk_x < xalign || surflevel->nblk_y < yalign) {
+            surflevel->mode = RADEON_SURF_MODE_1D;
             return;
         }
     }
-    surf->level[level].nblk_x  = ALIGN(surf->level[level].nblk_x, xalign);
-    surf->level[level].nblk_y  = ALIGN(surf->level[level].nblk_y, yalign);
-    surf->level[level].nblk_z  = ALIGN(surf->level[level].nblk_z, zalign);
+    surflevel->nblk_x  = ALIGN(surflevel->nblk_x, xalign);
+    surflevel->nblk_y  = ALIGN(surflevel->nblk_y, yalign);
+    surflevel->nblk_z  = ALIGN(surflevel->nblk_z, zalign);
 
-    surf->level[level].offset = offset;
-    surf->level[level].pitch_bytes = surf->level[level].nblk_x * surf->bpe * surf->nsamples;
-    surf->level[level].slice_size = surf->level[level].pitch_bytes * surf->level[level].nblk_y;
+    surflevel->offset = offset;
+    surflevel->pitch_bytes = surflevel->nblk_x * bpe * surf->nsamples;
+    surflevel->slice_size = surflevel->pitch_bytes * surflevel->nblk_y;
 
-    surf->bo_size = offset + surf->level[level].slice_size * surf->level[level].nblk_z * surf->array_size;
+    surf->bo_size = offset + surflevel->slice_size * surflevel->nblk_z * surf->array_size;
 }
 
 /* ===========================================================================
@@ -264,7 +265,7 @@ static int r6_surface_init_linear(struct radeon_surface_manager *surf_man,
     /* build mipmap tree */
     for (i = start_level; i <= surf->last_level; i++) {
         surf->level[i].mode = RADEON_SURF_MODE_LINEAR;
-        surf_minify(surf, i, xalign, yalign, zalign, offset);
+        surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset);
         /* level0 and first mipmap need to have alignment */
         offset = surf->bo_size;
         if ((i == 0)) {
@@ -292,7 +293,7 @@ static int r6_surface_init_linear_aligned(struct radeon_surface_manager *surf_ma
     /* build mipmap tree */
     for (i = start_level; i <= surf->last_level; i++) {
         surf->level[i].mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
-        surf_minify(surf, i, xalign, yalign, zalign, offset);
+        surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset);
         /* level0 and first mipmap need to have alignment */
         offset = surf->bo_size;
         if ((i == 0)) {
@@ -325,7 +326,7 @@ static int r6_surface_init_1d(struct radeon_surface_manager *surf_man,
     /* build mipmap tree */
     for (i = start_level; i <= surf->last_level; i++) {
         surf->level[i].mode = RADEON_SURF_MODE_1D;
-        surf_minify(surf, i, xalign, yalign, zalign, offset);
+        surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset);
         /* level0 and first mipmap need to have alignment */
         offset = surf->bo_size;
         if ((i == 0)) {
@@ -363,7 +364,7 @@ static int r6_surface_init_2d(struct radeon_surface_manager *surf_man,
     /* build mipmap tree */
     for (i = start_level; i <= surf->last_level; i++) {
         surf->level[i].mode = RADEON_SURF_MODE_2D;
-        surf_minify(surf, i, xalign, yalign, zalign, offset);
+        surf_minify(surf, surf->level+i, surf->bpe, i, xalign, yalign, zalign, offset);
         if (surf->level[i].mode == RADEON_SURF_MODE_1D) {
             return r6_surface_init_1d(surf_man, surf, offset, i);
         }
@@ -543,6 +544,8 @@ static int eg_init_hw_info(struct radeon_surface_manager *surf_man)
 }
 
 static void eg_surf_minify(struct radeon_surface *surf,
+                           struct radeon_surface_level *surflevel,
+                           unsigned bpe,
                            unsigned level,
                            unsigned slice_pt,
                            unsigned mtilew,
@@ -552,36 +555,38 @@ static void eg_surf_minify(struct radeon_surface *surf,
 {
     unsigned mtile_pr, mtile_ps;
 
-    surf->level[level].npix_x = mip_minify(surf->npix_x, level);
-    surf->level[level].npix_y = mip_minify(surf->npix_y, level);
-    surf->level[level].npix_z = mip_minify(surf->npix_z, level);
-    surf->level[level].nblk_x = (surf->level[level].npix_x + surf->blk_w - 1) / surf->blk_w;
-    surf->level[level].nblk_y = (surf->level[level].npix_y + surf->blk_h - 1) / surf->blk_h;
-    surf->level[level].nblk_z = (surf->level[level].npix_z + surf->blk_d - 1) / surf->blk_d;
-    if (surf->nsamples == 1 && surf->level[level].mode == RADEON_SURF_MODE_2D) {
-        if (surf->level[level].nblk_x < mtilew || surf->level[level].nblk_y < mtileh) {
-            surf->level[level].mode = RADEON_SURF_MODE_1D;
+    surflevel->npix_x = mip_minify(surf->npix_x, level);
+    surflevel->npix_y = mip_minify(surf->npix_y, level);
+    surflevel->npix_z = mip_minify(surf->npix_z, level);
+    surflevel->nblk_x = (surflevel->npix_x + surf->blk_w - 1) / surf->blk_w;
+    surflevel->nblk_y = (surflevel->npix_y + surf->blk_h - 1) / surf->blk_h;
+    surflevel->nblk_z = (surflevel->npix_z + surf->blk_d - 1) / surf->blk_d;
+    if (surf->nsamples == 1 && surflevel->mode == RADEON_SURF_MODE_2D) {
+        if (surflevel->nblk_x < mtilew || surflevel->nblk_y < mtileh) {
+            surflevel->mode = RADEON_SURF_MODE_1D;
             return;
         }
     }
-    surf->level[level].nblk_x  = ALIGN(surf->level[level].nblk_x, mtilew);
-    surf->level[level].nblk_y  = ALIGN(surf->level[level].nblk_y, mtileh);
-    surf->level[level].nblk_z  = ALIGN(surf->level[level].nblk_z, 1);
+    surflevel->nblk_x  = ALIGN(surflevel->nblk_x, mtilew);
+    surflevel->nblk_y  = ALIGN(surflevel->nblk_y, mtileh);
+    surflevel->nblk_z  = ALIGN(surflevel->nblk_z, 1);
 
     /* macro tile per row */
-    mtile_pr = surf->level[level].nblk_x / mtilew;
+    mtile_pr = surflevel->nblk_x / mtilew;
     /* macro tile per slice */
-    mtile_ps = (mtile_pr * surf->level[level].nblk_y) / mtileh;
+    mtile_ps = (mtile_pr * surflevel->nblk_y) / mtileh;
 
-    surf->level[level].offset = offset;
-    surf->level[level].pitch_bytes = surf->level[level].nblk_x * surf->bpe * slice_pt;
-    surf->level[level].slice_size = mtile_ps * mtileb * slice_pt;
+    surflevel->offset = offset;
+    surflevel->pitch_bytes = surflevel->nblk_x * bpe * slice_pt;
+    surflevel->slice_size = mtile_ps * mtileb * slice_pt;
 
-    surf->bo_size = offset + surf->level[level].slice_size * surf->level[level].nblk_z * surf->array_size;
+    surf->bo_size = offset + surflevel->slice_size * surflevel->nblk_z * surf->array_size;
 }
 
 static int eg_surface_init_1d(struct radeon_surface_manager *surf_man,
                               struct radeon_surface *surf,
+                              struct radeon_surface_level *level,
+                              unsigned bpe,
                               uint64_t offset, unsigned start_level)
 {
     uint32_t xalign, yalign, zalign, tilew;
@@ -589,45 +594,40 @@ static int eg_surface_init_1d(struct radeon_surface_manager *surf_man,
 
     /* compute alignment */
     tilew = 8;
-    xalign = surf_man->hw_info.group_bytes / (tilew * surf->bpe * surf->nsamples);
-    if (surf->flags & RADEON_SURF_SBUFFER) {
-        xalign = surf_man->hw_info.group_bytes / (tilew * surf->nsamples);
-    }
+    xalign = surf_man->hw_info.group_bytes / (tilew * bpe * surf->nsamples);
     xalign = MAX2(tilew, xalign);
     yalign = tilew;
     zalign = 1;
     if (surf->flags & RADEON_SURF_SCANOUT) {
-        xalign = MAX2((surf->bpe == 1) ? 64 : 32, xalign);
+        xalign = MAX2((bpe == 1) ? 64 : 32, xalign);
     }
+
     if (!start_level) {
-        surf->bo_alignment = MAX2(256, surf_man->hw_info.group_bytes);
+        unsigned alignment = MAX2(256, surf_man->hw_info.group_bytes);
+        surf->bo_alignment = MAX2(surf->bo_alignment, alignment);
+
+        if (offset) {
+            offset = ALIGN(offset, alignment);
+        }
     }
 
     /* build mipmap tree */
     for (i = start_level; i <= surf->last_level; i++) {
-        surf->level[i].mode = RADEON_SURF_MODE_1D;
-        surf_minify(surf, i, xalign, yalign, zalign, offset);
+        level[i].mode = RADEON_SURF_MODE_1D;
+        surf_minify(surf, level+i, bpe, i, xalign, yalign, zalign, offset);
         /* level0 and first mipmap need to have alignment */
         offset = surf->bo_size;
         if ((i == 0)) {
             offset = ALIGN(offset, surf->bo_alignment);
         }
     }
-
-    /* The depth and stencil buffers are in separate resources on evergreen.
-     * We allocate them in one buffer next to each other to simplify
-     * communication between the DDX and the Mesa driver. */
-    if ((surf->flags & (RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) ==
-	(RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) {
-        surf->stencil_offset = ALIGN(surf->bo_size, surf->bo_alignment);
-        surf->bo_size = surf->stencil_offset + surf->bo_size / 4;
-    }
-
     return 0;
 }
 
 static int eg_surface_init_2d(struct radeon_surface_manager *surf_man,
                               struct radeon_surface *surf,
+                              struct radeon_surface_level *level,
+                              unsigned bpe, unsigned tile_split,
                               uint64_t offset, unsigned start_level)
 {
     unsigned tilew, tileh, tileb;
@@ -638,11 +638,11 @@ static int eg_surface_init_2d(struct radeon_surface_manager *surf_man,
     /* compute tile values */
     tilew = 8;
     tileh = 8;
-    tileb = tilew * tileh * surf->bpe * surf->nsamples;
+    tileb = tilew * tileh * bpe * surf->nsamples;
     /* slices per tile */
     slice_pt = 1;
-    if (tileb > surf->tile_split) {
-        slice_pt = tileb / surf->tile_split;
+    if (tileb > tile_split) {
+        slice_pt = tileb / tile_split;
     }
     tileb = tileb / slice_pt;
 
@@ -653,15 +653,20 @@ static int eg_surface_init_2d(struct radeon_surface_manager *surf_man,
     mtileb = (mtilew / tilew) * (mtileh / tileh) * tileb;
 
     if (!start_level) {
-        surf->bo_alignment = MAX2(256, mtileb);
+        unsigned alignment = MAX2(256, mtileb);
+        surf->bo_alignment = MAX2(surf->bo_alignment, alignment);
+
+        if (offset) {
+            offset = ALIGN(offset, alignment);
+        }
     }
 
     /* build mipmap tree */
     for (i = start_level; i <= surf->last_level; i++) {
-        surf->level[i].mode = RADEON_SURF_MODE_2D;
-        eg_surf_minify(surf, i, slice_pt, mtilew, mtileh, mtileb, offset);
-        if (surf->level[i].mode == RADEON_SURF_MODE_1D) {
-            return eg_surface_init_1d(surf_man, surf, offset, i);
+        level[i].mode = RADEON_SURF_MODE_2D;
+        eg_surf_minify(surf, level+i, bpe, i, slice_pt, mtilew, mtileh, mtileb, offset);
+        if (level[i].mode == RADEON_SURF_MODE_1D) {
+            return eg_surface_init_1d(surf_man, surf, level, bpe, offset, i);
         }
         /* level0 and first mipmap need to have alignment */
         offset = surf->bo_size;
@@ -669,13 +674,6 @@ static int eg_surface_init_2d(struct radeon_surface_manager *surf_man,
             offset = ALIGN(offset, surf->bo_alignment);
         }
     }
-
-    if ((surf->flags & (RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) ==
-	(RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER)) {
-        surf->stencil_offset = ALIGN(surf->bo_size, surf->bo_alignment);
-        surf->bo_size = surf->stencil_offset + surf->bo_size / 4;
-    }
-
     return 0;
 }
 
@@ -762,6 +760,51 @@ static int eg_surface_sanity(struct radeon_surface_manager *surf_man,
     return 0;
 }
 
+static int eg_surface_init_1d_miptrees(struct radeon_surface_manager *surf_man,
+                                       struct radeon_surface *surf)
+{
+    unsigned zs_flags = RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER;
+    int r, is_depth_stencil = (surf->flags & zs_flags) == zs_flags;
+    /* Old libdrm headers didn't have stencil_level in it. This prevents crashes. */
+    struct radeon_surface_level tmp[RADEON_SURF_MAX_LEVEL];
+    struct radeon_surface_level *stencil_level =
+        (surf->flags & RADEON_SURF_HAS_SBUFFER_MIPTREE) ? surf->stencil_level : tmp;
+
+    r = eg_surface_init_1d(surf_man, surf, surf->level, surf->bpe, 0, 0);
+    if (r)
+        return r;
+
+    if (is_depth_stencil) {
+        r = eg_surface_init_1d(surf_man, surf, stencil_level, 1,
+                               surf->bo_size, 0);
+        surf->stencil_offset = stencil_level[0].offset;
+    }
+    return r;
+}
+
+static int eg_surface_init_2d_miptrees(struct radeon_surface_manager *surf_man,
+                                       struct radeon_surface *surf)
+{
+    unsigned zs_flags = RADEON_SURF_ZBUFFER | RADEON_SURF_SBUFFER;
+    int r, is_depth_stencil = (surf->flags & zs_flags) == zs_flags;
+    /* Old libdrm headers didn't have stencil_level in it. This prevents crashes. */
+    struct radeon_surface_level tmp[RADEON_SURF_MAX_LEVEL];
+    struct radeon_surface_level *stencil_level =
+        (surf->flags & RADEON_SURF_HAS_SBUFFER_MIPTREE) ? surf->stencil_level : tmp;
+
+    r = eg_surface_init_2d(surf_man, surf, surf->level, surf->bpe,
+                           surf->tile_split, 0, 0);
+    if (r)
+        return r;
+
+    if (is_depth_stencil) {
+        r = eg_surface_init_2d(surf_man, surf, stencil_level, 1,
+                               surf->stencil_tile_split, surf->bo_size, 0);
+        surf->stencil_offset = stencil_level[0].offset;
+    }
+    return r;
+}
+
 static int eg_surface_init(struct radeon_surface_manager *surf_man,
                            struct radeon_surface *surf)
 {
@@ -797,6 +840,7 @@ static int eg_surface_init(struct radeon_surface_manager *surf_man,
     }
 
     surf->stencil_offset = 0;
+    surf->bo_alignment = 0;
 
     /* check tiling mode */
     switch (mode) {
@@ -807,10 +851,10 @@ static int eg_surface_init(struct radeon_surface_manager *surf_man,
         r = r6_surface_init_linear_aligned(surf_man, surf, 0, 0);
         break;
     case RADEON_SURF_MODE_1D:
-        r = eg_surface_init_1d(surf_man, surf, 0, 0);
+        r = eg_surface_init_1d_miptrees(surf_man, surf);
         break;
     case RADEON_SURF_MODE_2D:
-        r = eg_surface_init_2d(surf_man, surf, 0, 0);
+        r = eg_surface_init_2d_miptrees(surf_man, surf);
         break;
     default:
         return -EINVAL;
@@ -1058,6 +1102,7 @@ static int si_surface_init(struct radeon_surface_manager *surf_man,
     }
 
     surf->stencil_offset = 0;
+    surf->bo_alignment = 0;
 
     /* check tiling mode */
     switch (mode) {
@@ -1068,10 +1113,10 @@ static int si_surface_init(struct radeon_surface_manager *surf_man,
         r = si_surface_init_linear_aligned(surf_man, surf, 0, 0);
         break;
     case RADEON_SURF_MODE_1D:
-        r = eg_surface_init_1d(surf_man, surf, 0, 0);
+        r = eg_surface_init_1d_miptrees(surf_man, surf);
         break;
     case RADEON_SURF_MODE_2D:
-        r = eg_surface_init_2d(surf_man, surf, 0, 0);
+        r = eg_surface_init_2d_miptrees(surf_man, surf);
         break;
     default:
         return -EINVAL;
diff --git a/radeon/radeon_surface.h b/radeon/radeon_surface.h
index bfee8ab0..ed4b9941 100644
--- a/radeon/radeon_surface.h
+++ b/radeon/radeon_surface.h
@@ -54,6 +54,7 @@
 #define RADEON_SURF_SCANOUT                     (1 << 16)
 #define RADEON_SURF_ZBUFFER                     (1 << 17)
 #define RADEON_SURF_SBUFFER                     (1 << 18)
+#define RADEON_SURF_HAS_SBUFFER_MIPTREE         (1 << 19)
 
 #define RADEON_SURF_GET(v, field)   (((v) >> RADEON_SURF_ ## field ## _SHIFT) & RADEON_SURF_ ## field ## _MASK)
 #define RADEON_SURF_SET(v, field)   (((v) & RADEON_SURF_ ## field ## _MASK) << RADEON_SURF_ ## field ## _SHIFT)
@@ -102,6 +103,7 @@ struct radeon_surface {
     uint32_t                    stencil_tile_split;
     uint64_t                    stencil_offset;
     struct radeon_surface_level level[RADEON_SURF_MAX_LEVEL];
+    struct radeon_surface_level stencil_level[RADEON_SURF_MAX_LEVEL];
 };
 
 struct radeon_surface_manager *radeon_surface_manager_new(int fd);