Softer shadows in raytracing and RTAO

code/CMakeLists.txt

@@ -77,6 +77,7 @@ TARGET_LINK_LIBRARIES(code PUBLIC jansson)
 target_link_libraries(code PUBLIC anl)
 
 target_link_libraries(code PUBLIC imgui)
+target_link_libraries(code PUBLIC implot)
 
 IF(FSO_BUILD_WITH_OPENXR)
 	target_link_libraries(code PUBLIC OpenXR::openxr_loader)

code/cmdline/cmdline.cpp

@@ -2409,10 +2409,9 @@ bool SetCmdlineParams()
 	if (gr_sync_validation_arg.found()) {
 		// Enables the Vulkan validation layer + debug messenger + synchronization
 		// validation feature (see VulkanRenderer::initializeInstance), but NOT the
-		// engine-level graphics debug output: -gr_debug additionally draws debug
-		// overlays (e.g. output_uniform_debug_data), which are unrelated to GPU
-		// sync validation and would otherwise appear as spurious on-screen
-		// artifacts. Keep this a pure GPU-validation switch.
+		// engine-level graphics debug output: -gr_debug additionally feeds extra
+		// stat groups into the ImGui frame profiler overlay (see gr_get_debug_stats),
+		// which are unrelated to GPU sync validation. Keep this a pure GPU-validation switch.
 		Cmdline_gr_sync_validation = true;
 	}
 

code/def_files/data/effects/deferred-f.sdr

@@ -18,13 +18,14 @@ layout(set = 1, binding = 1) uniform sampler2D sTextures[16];
 #define SpecBuffer sTextures[3]
 
 layout(set = 0, binding = 2) uniform sampler2DArrayShadow shadow_map;
+layout(set = 0, binding = 7) uniform sampler2DArray shadow_map_raw;
 
 #ifdef ENV_MAP
 layout(set = 0, binding = 3) uniform samplerCube sEnvmap;
 layout(set = 0, binding = 4) uniform samplerCube sIrrmap;
 #endif
 
-#ifdef RT_SHADOWS
+#if defined(RT_SHADOWS) || defined(RTAO)
 layout(set = 0, binding = 5) uniform accelerationStructureEXT shadow_tlas;
 #endif
 #else
@@ -33,6 +34,7 @@ uniform sampler2D NormalBuffer;
 uniform sampler2D PositionBuffer;
 uniform sampler2D SpecBuffer;
 uniform sampler2DArrayShadow shadow_map;
+uniform sampler2DArray shadow_map_raw;
 
 #ifdef ENV_MAP
 uniform samplerCube sEnvmap;
@@ -63,11 +65,16 @@ uniform shadowCascadeParams {
 	int cascade_count;
 	float rtShadowBiasMin;
 	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
 	vec4 cascade_distances[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 	vec4 smoothness_factors[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+	vec4 penumbra_scale[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 };
 
 #ifdef VULKAN
@@ -291,6 +298,17 @@ void main()
 
 	if (lightType == LT_AMBIENT) {
 		float ao = position_buffer.w;
+#ifdef RTAO
+		{
+			// Raytraced AO multiplies into the same ao term as the baked AO maps,
+			// so the env-map/IBL ambient below darkens consistently for free.
+			vec3 worldPos = (inv_view_matrix * vec4(position, 1.0)).xyz;
+			vec3 worldNormal = normalize((inv_view_matrix * vec4(normal, 0.0)).xyz);
+			float rtaoBias = computeRtShadowBias(length(position), rtShadowBiasMin, rtShadowBiasMax);
+			ao *= traceAmbientOcclusion(shadow_tlas, worldPos, worldNormal,
+				rtaoRadius, rtaoStrength, rtaoSampleCount, rtaoBias, gl_FragCoord.xy);
+		}
+#endif
 		fragmentColor.rgb = diffuseLightColor * diffColor * ao;
 
 #ifdef ENV_MAP
@@ -315,15 +333,22 @@ void main()
 			vec3 worldLightDir = normalize((inv_view_matrix * vec4(lightDir, 0.0)).xyz);
 
 			float rtShadowBias = computeRtShadowBias(length(position), rtShadowBiasMin, rtShadowBiasMax);
-			attenuation *= traceShadowRay(shadow_tlas, worldPos, worldNormal, worldLightDir, lightDist, rtShadowBias);
+			// Penumbra cone: for directional lights sourceRadius IS the tangent of the
+			// sun's angular radius ($SunAngularSize); for local lights the subtended
+			// half-angle follows from the physical source radius and the light distance.
+			float coneTan = (lightType == LT_DIRECTIONAL)
+				? sourceRadius
+				: sourceRadius / max(lightDist, 1e-3);
+			attenuation *= traceShadowRayCone(shadow_tlas, worldPos, worldNormal, worldLightDir, lightDist,
+				rtShadowBias, coneTan, rtShadowSampleCount, gl_FragCoord.xy);
 #else
 			vec4 fragShadowPos = shadow_mv_matrix * inv_view_matrix * vec4(position, 1.0);
 			vec4 fragShadowUV[NUM_SHADOW_CASCADES];
 			for (int i = 0; i < NUM_SHADOW_CASCADES; i++) {
 				fragShadowUV[i] = transformToShadowMap(shadow_proj_matrix[i], i, fragShadowPos);
 			}
 
-			attenuation *= getShadowValue(shadow_map, -position.z, fragShadowUV, cascade_distances, smoothness_factors, cascade_offset, cascade_count);
+			attenuation *= getShadowValue(shadow_map, shadow_map_raw, -position.z, fragShadowUV, cascade_distances, smoothness_factors, penumbra_scale, cascade_offset, cascade_count);
 #endif
 		}
 

code/def_files/data/effects/main-f.sdr

@@ -101,11 +101,16 @@ uniform shadowCascadeParams {
 	int cascade_count;
 	float rtShadowBiasMin;
 	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
 	vec4 cascade_distances[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 	vec4 smoothness_factors[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+	vec4 penumbra_scale[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 };
 
 #ifdef VULKAN
@@ -118,6 +123,7 @@ layout(set = 1, binding = 1) uniform sampler2DArray materialTextures[16];
 #define sMiscmap materialTextures[6]
 
 layout(set = 0, binding = 2) uniform sampler2DArrayShadow shadow_map;
+layout(set = 0, binding = 7) uniform sampler2DArray shadow_map_raw;
 layout(set = 1, binding = 5) uniform sampler2D sFramebuffer;
 
 #ifdef MODEL_SDR_FLAG_RT_SHADOWS
@@ -164,6 +170,7 @@ uniform sampler2DArray sMiscmap;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_MISC
 #prereplace IF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
 uniform sampler2DArrayShadow shadow_map;
+uniform sampler2DArray shadow_map_raw;
 #prereplace ENDIF_FLAG_COMPILED MODEL_SDR_FLAG_SHADOWS
 
 in VertexOutput {
@@ -248,7 +255,11 @@ vec3 CalculateLighting(vec3 normal, vec3 diffuseMaterial, vec3 specularMaterial,
 		if (rtShadowsActive && lights[i].light_type == LT_DIRECTIONAL && shadowedDirectionalCount < MAX_RT_SHADOW_LIGHTS) {
 			vec3 worldSunDir = normalize((invView * vec4(lights[i].position.xyz, 0.0)).xyz);
 			float rtShadowBias = computeRtShadowBias(length(vertIn.position.xyz), rtShadowBiasMin, rtShadowBiasMax);
-			shadow = traceShadowRay(shadow_tlas, worldPos, worldNormal, worldSunDir, RT_SHADOW_MAX_DISTANCE, rtShadowBias);
+			// For directional lights ml_sourceRadius is the tangent of the sun's
+			// angular radius ($SunAngularSize) -- 0 keeps hard shadows.
+			shadow = traceShadowRayCone(shadow_tlas, worldPos, worldNormal, worldSunDir, RT_SHADOW_MAX_DISTANCE,
+				rtShadowBias, lights[i].ml_sourceRadius,
+				rtShadowSampleCount, gl_FragCoord.xy);
 			++shadowedDirectionalCount;
 		} else {
 			shadow = 1.0;
@@ -410,7 +421,7 @@ void main()
 			// up to MAX_RT_SHADOW_LIGHTS) inside CalculateLighting()'s loop instead --
 			// this single shared `shadow` value only matters for the CSM fallback below.
 			#ifndef MODEL_SDR_FLAG_RT_SHADOWS
-				shadow = getShadowValue(shadow_map, -vertIn.position.z, vertIn.shadowUV, cascade_distances, smoothness_factors, cascade_offset, cascade_count);
+				shadow = getShadowValue(shadow_map, shadow_map_raw, -vertIn.position.z, vertIn.shadowUV, cascade_distances, smoothness_factors, penumbra_scale, cascade_offset, cascade_count);
 			#endif
 			#prereplace ENDIF_FLAG //MODEL_SDR_FLAG_SHADOWS
 			baseColor.rgb = CalculateLighting(normal, baseColor.rgb, specColor.rgb, glossData, fresnelFactor, shadow, aoFactors.x);

code/def_files/data/effects/main-v.sdr

@@ -115,11 +115,16 @@ uniform shadowCascadeParams {
 	int cascade_count;
 	float rtShadowBiasMin;
 	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
 	vec4 cascade_distances[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 	vec4 smoothness_factors[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+	vec4 penumbra_scale[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 };
 
 #ifdef VULKAN

code/def_files/data/effects/shadow_map-g.sdr

@@ -7,16 +7,24 @@ layout (triangle_strip, max_vertices = 3) out;
 layout(invocations = NUM_SHADOW_CASCADES) in;
 #endif
 
-layout (std140) uniform shadowCascadeParams {
-    int cascade_offset;
-    int cascade_count;
-    float rtShadowBiasMin;
-    float rtShadowBiasMax;
+// OpenGL-only: the Vulkan backend has no geometry-shader stage, and disables the
+// shadow pass outright when it can't write gl_Layer from the vertex shader, so it
+// never requests GEOMETRY_FALLBACK. Hence no Vulkan set/binding qualifier here.
+layout(std140) uniform shadowCascadeParams {
+	int cascade_offset;
+	int cascade_count;
+	float rtShadowBiasMin;
+	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
 	vec4 cascade_distances[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 	vec4 smoothness_factors[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+	vec4 penumbra_scale[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 };
 
 in VertexOutput {

code/def_files/data/effects/shadow_map-v.sdr

@@ -38,15 +38,20 @@ layout(set = 0, binding = 6, std140)
 layout(std140)
 #endif
 uniform shadowCascadeParams {
-    int cascade_offset;
-    int cascade_count;
-    float rtShadowBiasMin;
-    float rtShadowBiasMax;
+	int cascade_offset;
+	int cascade_count;
+	float rtShadowBiasMin;
+	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
 	vec4 cascade_distances[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 	vec4 smoothness_factors[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
+	vec4 penumbra_scale[(NUM_SHADOW_CASCADES + 4 - 1) / 4];
 };
 
 #ifdef VULKAN