frustum_debug.slang

#include "shared.h"
// set to 1 to compare against the inverse view-projection reference
#define VISUALIZE_REFERENCE 0

struct Frustum {
    vec4 planes[6]; // left, right, top, bottom, near, far
};

// clang-format off
static const vec4 FRUSTUM_CORNERS_CS[8] = {
    vec4(-1, -1, 1,    1),
    vec4( 1, -1, 1,    1),
    vec4( 1,  1, 1,    1),
    vec4(-1,  1, 1,    1),

    vec4(-1, -1, 1e-6, 1),
    vec4( 1, -1, 1e-6, 1),
    vec4( 1,  1, 1e-6, 1),
    vec4(-1,  1, 1e-6, 1),
};

// line list indices
static const uint FRUSTUM_EDGES[24] = {
    0,1, 1,2, 2,3, 3,0,
    4,5, 5,6, 6,7, 7,4,
    0,4, 1,5, 2,6, 3,7
};

// which 3 planes form each corner
static const uint CORNER_PLANES[24] = {
    4,2,0, 4,2,1,
    4,3,1, 4,3,0,

    5,2,0, 5,2,1,
    5,3,1, 5,3,0,
};
// clang-format on

[vk::push_constant]
FrustumDebugPushConstants pc;

// intersect 3 planes -> world-space point
vec4 intersect_three_planes(vec4 p0, vec4 p1, vec4 p2) {
    vec3 n0 = p0.xyz;
    vec3 n1 = p1.xyz;
    vec3 n2 = p2.xyz;

    f32 d0 = p0.w;
    f32 d1 = p1.w;
    f32 d2 = p2.w;

    vec3 cross12 = cross(n1, n2);
    f32 det = dot(n0, cross12);

    if (abs(det) < 1e-10)
        return vec4(0);

    vec3 point = -(d0 * cross12 + d1 * cross(n2, n0) + d2 * cross(n0, n1)) / det;

    return vec4(point, 1.0);
}

Frustum extract_frustum_(mat4 view_proj) {
    Frustum f;

    mat4 rows = transpose(view_proj);

    f.planes[0] = rows[3] + rows[0]; // left
    f.planes[1] = rows[3] - rows[0]; // right
    f.planes[2] = rows[3] + rows[1]; // bottom
    f.planes[3] = rows[3] - rows[1]; // top
    f.planes[4] = rows[3] - rows[2]; // near (reverse-z)

    // placeholder, replaced with a far plane later
    f.planes[5] = rows[2];

    for (uint i = 0; i < 6; ++i)
        f.planes[i] /= length(f.planes[i].xyz);

    return f;
}

void build_culling_far_plane(inout Frustum frustum, FrameUBO frame) {
    vec4 near_ws = vec4(0, 0, 1.0, 1.0) * pc.debug_inv_view_proj;
    vec4 far_ws = vec4(0, 0, 1e-6, 1.0) * pc.debug_inv_view_proj;

    near_ws /= near_ws.w;
    far_ws /= far_ws.w;

    vec3 forward = normalize(far_ws.xyz - near_ws.xyz);

    // vec3 forward = vec3(0.0, 1.0, 0.0);

    // todo: use pc.max_draw_distance
    vec3 far_point = frame.position_ws.xyz + forward * 10000.0;

    vec3 normal = -forward;

    frustum.planes[5] = vec4(normal, -dot(normal, far_point));
}

// reference clip-space corner with the inverse view-projection matrix
vec4 reconstruct_reference_corner(uint corner) {
    vec4 world = FRUSTUM_CORNERS_CS[corner] * pc.debug_inv_view_proj;
    return world / world.w;
}

// same as reconstruct_reference_corner, but using the extracted frustum planes
vec4 reconstruct_extracted_corner(Frustum frustum, uint corner) {
    uint base = corner * 3;

    return intersect_three_planes(
        frustum.planes[CORNER_PLANES[base + 0]],
        frustum.planes[CORNER_PLANES[base + 1]],
        frustum.planes[CORNER_PLANES[base + 2]]
    );
}

[shader("vertex")]
vec4 vsMain(uint vertex_id: SV_VertexID) : SV_Position {
    FrameUBO frame = uniform_buffers[pc.frame_ubo_idx];

    uint corner = FRUSTUM_EDGES[vertex_id];

#if VISUALIZE_REFERENCE

    vec4 world = reconstruct_reference_corner(corner);

#else

    Frustum frustum = extract_frustum_(pc.debug_view_proj);

    build_culling_far_plane(frustum, frame);

    vec4 world = reconstruct_extracted_corner(frustum, corner);

#endif

    return (world * frame.view_proj);
}

[shader("fragment")]
vec4 fsMain() : SV_Target {
#if VISUALIZE_REFERENCE
    return vec4(100.0, 0.0, 100.0, 1.0);
#endif
    return vec4(0.0, 100.00, 100.0, 10.0);
}