diff --git a/wgpu-core/src/binding_model.rs b/wgpu-core/src/binding_model.rs index a64fe78465..68300706a1 100644 --- a/wgpu-core/src/binding_model.rs +++ b/wgpu-core/src/binding_model.rs @@ -8,7 +8,7 @@ use crate::{ init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction}, resource::{ DestroyedResourceError, MissingBufferUsageError, MissingTextureUsageError, ParentDevice, - Resource, ResourceInfo, ResourceType, + Resource, ResourceErrorIdent, ResourceInfo, ResourceType, }, resource_log, snatch::{SnatchGuard, Snatchable}, @@ -771,19 +771,21 @@ pub enum BindingResource<'a> { #[non_exhaustive] pub enum BindError { #[error( - "Bind group {group} expects {expected} dynamic offset{s0}. However {actual} dynamic offset{s1} were provided.", + "{bind_group} {group} expects {expected} dynamic offset{s0}. However {actual} dynamic offset{s1} were provided.", s0 = if *.expected >= 2 { "s" } else { "" }, s1 = if *.actual >= 2 { "s" } else { "" }, )] MismatchedDynamicOffsetCount { + bind_group: ResourceErrorIdent, group: u32, actual: usize, expected: usize, }, #[error( - "Dynamic binding index {idx} (targeting bind group {group}, binding {binding}) with value {offset}, does not respect device's requested `{limit_name}` limit: {alignment}" + "Dynamic binding index {idx} (targeting {bind_group} {group}, binding {binding}) with value {offset}, does not respect device's requested `{limit_name}` limit: {alignment}" )] UnalignedDynamicBinding { + bind_group: ResourceErrorIdent, idx: usize, group: u32, binding: u32, @@ -792,10 +794,11 @@ pub enum BindError { limit_name: &'static str, }, #[error( - "Dynamic binding offset index {idx} with offset {offset} would overrun the buffer bound to bind group {group} -> binding {binding}. \ + "Dynamic binding offset index {idx} with offset {offset} would overrun the buffer bound to {bind_group} {group} -> binding {binding}. \ Buffer size is {buffer_size} bytes, the binding binds bytes {binding_range:?}, meaning the maximum the binding can be offset is {maximum_dynamic_offset} bytes", )] DynamicBindingOutOfBounds { + bind_group: ResourceErrorIdent, idx: usize, group: u32, binding: u32, @@ -892,10 +895,10 @@ impl BindGroup { &self, bind_group_index: u32, offsets: &[wgt::DynamicOffset], - limits: &wgt::Limits, ) -> Result<(), BindError> { if self.dynamic_binding_info.len() != offsets.len() { return Err(BindError::MismatchedDynamicOffsetCount { + bind_group: self.error_ident(), group: bind_group_index, expected: self.dynamic_binding_info.len(), actual: offsets.len(), @@ -908,9 +911,11 @@ impl BindGroup { .zip(offsets.iter()) .enumerate() { - let (alignment, limit_name) = buffer_binding_type_alignment(limits, info.binding_type); + let (alignment, limit_name) = + buffer_binding_type_alignment(&self.device.limits, info.binding_type); if offset as wgt::BufferAddress % alignment as u64 != 0 { return Err(BindError::UnalignedDynamicBinding { + bind_group: self.error_ident(), group: bind_group_index, binding: info.binding_idx, idx, @@ -922,6 +927,7 @@ impl BindGroup { if offset as wgt::BufferAddress > info.maximum_dynamic_offset { return Err(BindError::DynamicBindingOutOfBounds { + bind_group: self.error_ident(), group: bind_group_index, binding: info.binding_idx, idx, diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 5639847ceb..5f913fd791 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -79,15 +79,15 @@ index format changes. #![allow(clippy::reversed_empty_ranges)] use crate::{ - binding_model::{buffer_binding_type_alignment, BindGroup, BindGroupLayout, PipelineLayout}, + binding_model::{BindError, BindGroup, BindGroupLayout, PipelineLayout}, command::{ BasePass, BindGroupStateChange, ColorAttachmentError, DrawError, MapPassErr, PassErrorScope, RenderCommandError, StateChange, }, conv, device::{ - AttachmentData, Device, DeviceError, MissingDownlevelFlags, - RenderPassCompatibilityCheckType, RenderPassContext, SHADER_STAGE_COUNT, + AttachmentData, Device, DeviceError, MissingDownlevelFlags, RenderPassContext, + SHADER_STAGE_COUNT, }, error::{ErrorFormatter, PrettyError}, hal_api::HalApi, @@ -366,9 +366,14 @@ impl RenderBundleEncoder { vertex: (0..hal::MAX_VERTEX_BUFFERS).map(|_| None).collect(), index: None, flat_dynamic_offsets: Vec::new(), + device: device.clone(), + commands: Vec::new(), + buffer_memory_init_actions: Vec::new(), + texture_memory_init_actions: Vec::new(), + next_dynamic_offset: 0, }; - let indices = &device.tracker_indices; + let indices = &state.device.tracker_indices; state .trackers .buffers @@ -395,12 +400,6 @@ impl RenderBundleEncoder { .write() .set_size(indices.query_sets.size()); - let mut commands = Vec::new(); - let mut buffer_memory_init_actions = Vec::new(); - let mut texture_memory_init_actions = Vec::new(); - - let mut next_dynamic_offset = 0; - let base = &self.base; for &command in &base.commands { @@ -410,114 +409,28 @@ impl RenderBundleEncoder { num_dynamic_offsets, bind_group_id, } => { - let scope = PassErrorScope::SetBindGroup(bind_group_id); - - let bind_group = bind_group_guard - .get(bind_group_id) - .map_err(|_| RenderCommandError::InvalidBindGroupId(bind_group_id)) - .map_pass_err(scope)?; - - state - .trackers - .bind_groups - .write() - .add_single(bind_group); - - bind_group.same_device(device).map_pass_err(scope)?; - - let max_bind_groups = device.limits.max_bind_groups; - if index >= max_bind_groups { - return Err(RenderCommandError::BindGroupIndexOutOfRange { - index, - max: max_bind_groups, - }) - .map_pass_err(scope); - } - - // Identify the next `num_dynamic_offsets` entries from `base.dynamic_offsets`. - let offsets_range = - next_dynamic_offset..next_dynamic_offset + num_dynamic_offsets; - next_dynamic_offset = offsets_range.end; - let offsets = &base.dynamic_offsets[offsets_range.clone()]; - - if bind_group.dynamic_binding_info.len() != offsets.len() { - return Err(RenderCommandError::InvalidDynamicOffsetCount { - actual: offsets.len(), - expected: bind_group.dynamic_binding_info.len(), - }) - .map_pass_err(scope); - } - - // Check for misaligned offsets. - for (offset, info) in offsets - .iter() - .map(|offset| *offset as wgt::BufferAddress) - .zip(bind_group.dynamic_binding_info.iter()) - { - let (alignment, limit_name) = - buffer_binding_type_alignment(&device.limits, info.binding_type); - if offset % alignment as u64 != 0 { - return Err(RenderCommandError::UnalignedBufferOffset( - offset, limit_name, alignment, - )) - .map_pass_err(scope); - } - } - - buffer_memory_init_actions.extend_from_slice(&bind_group.used_buffer_ranges); - texture_memory_init_actions.extend_from_slice(&bind_group.used_texture_ranges); - - state.set_bind_group(index, bind_group_guard.get(bind_group_id).as_ref().unwrap(), &bind_group.layout, offsets_range); - unsafe { - state - .trackers - .merge_bind_group(&bind_group.used) - .map_pass_err(scope)? - }; - //Note: stateless trackers are not merged: the lifetime reference - // is held to the bind group itself. + let scope = PassErrorScope::SetBindGroup; + set_bind_group( + &mut state, + &bind_group_guard, + &base.dynamic_offsets, + index, + num_dynamic_offsets, + bind_group_id, + ) + .map_pass_err(scope)?; } RenderCommand::SetPipeline(pipeline_id) => { - let scope = PassErrorScope::SetPipelineRender(pipeline_id); - - let pipeline = pipeline_guard - .get(pipeline_id) - .map_err(|_| RenderCommandError::InvalidPipeline(pipeline_id)) - .map_pass_err(scope)?; - - state - .trackers - .render_pipelines - .write() - .add_single(pipeline); - - pipeline.same_device(device).map_pass_err(scope)?; - - self.context - .check_compatible(&pipeline.pass_context, RenderPassCompatibilityCheckType::RenderPipeline) - .map_err(RenderCommandError::IncompatiblePipelineTargets) - .map_pass_err(scope)?; - - if (pipeline.flags.contains(PipelineFlags::WRITES_DEPTH) - && self.is_depth_read_only) - || (pipeline.flags.contains(PipelineFlags::WRITES_STENCIL) - && self.is_stencil_read_only) - { - return Err(RenderCommandError::IncompatiblePipelineRods) - .map_pass_err(scope); - } - - let pipeline_state = PipelineState::new(pipeline, pipeline_id); - - commands.push(ArcRenderCommand::SetPipeline(pipeline.clone())); - - // If this pipeline uses push constants, zero out their values. - if let Some(iter) = pipeline_state.zero_push_constants() { - commands.extend(iter) - } - - state.invalidate_bind_groups(&pipeline_state, &pipeline.layout); - state.pipeline = Some(pipeline_state); + let scope = PassErrorScope::SetPipelineRender; + set_pipeline( + &mut state, + &pipeline_guard, + &self.context, + self.is_depth_read_only, + self.is_stencil_read_only, + pipeline_id, + ) + .map_pass_err(scope)?; } RenderCommand::SetIndexBuffer { buffer_id, @@ -525,33 +438,16 @@ impl RenderBundleEncoder { offset, size, } => { - let scope = PassErrorScope::SetIndexBuffer(buffer_id); - - let buffer = buffer_guard - .get(buffer_id) - .map_err(|_| RenderCommandError::InvalidBufferId(buffer_id)) - .map_pass_err(scope)?; - - state - .trackers - .buffers - .write() - .merge_single(buffer, hal::BufferUses::INDEX) - .map_pass_err(scope)?; - - buffer.same_device(device).map_pass_err(scope)?; - buffer.check_usage(wgt::BufferUsages::INDEX).map_pass_err(scope)?; - - let end = match size { - Some(s) => offset + s.get(), - None => buffer.size, - }; - buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( - buffer, - offset..end, - MemoryInitKind::NeedsInitializedMemory, - )); - state.set_index_buffer(buffer.clone(), index_format, offset..end); + let scope = PassErrorScope::SetIndexBuffer; + set_index_buffer( + &mut state, + &buffer_guard, + buffer_id, + index_format, + offset, + size, + ) + .map_pass_err(scope)?; } RenderCommand::SetVertexBuffer { slot, @@ -559,41 +455,9 @@ impl RenderBundleEncoder { offset, size, } => { - let scope = PassErrorScope::SetVertexBuffer(buffer_id); - - let max_vertex_buffers = device.limits.max_vertex_buffers; - if slot >= max_vertex_buffers { - return Err(RenderCommandError::VertexBufferIndexOutOfRange { - index: slot, - max: max_vertex_buffers, - }) - .map_pass_err(scope); - } - - let buffer = buffer_guard - .get(buffer_id) - .map_err(|_| RenderCommandError::InvalidBufferId(buffer_id)) + let scope = PassErrorScope::SetVertexBuffer; + set_vertex_buffer(&mut state, &buffer_guard, slot, buffer_id, offset, size) .map_pass_err(scope)?; - - state - .trackers - .buffers.write() - .merge_single(buffer, hal::BufferUses::VERTEX) - .map_pass_err(scope)?; - - buffer.same_device(device).map_pass_err(scope)?; - buffer.check_usage(wgt::BufferUsages::VERTEX).map_pass_err(scope)?; - - let end = match size { - Some(s) => offset + s.get(), - None => buffer.size, - }; - buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( - buffer, - offset..end, - MemoryInitKind::NeedsInitializedMemory, - )); - state.vertex[slot as usize] = Some(VertexState::new(buffer.clone(), offset..end)); } RenderCommand::SetPushConstant { stages, @@ -602,15 +466,8 @@ impl RenderBundleEncoder { values_offset, } => { let scope = PassErrorScope::SetPushConstant; - let end_offset = offset + size_bytes; - - let pipeline_state = state.pipeline(scope)?; - - pipeline_state.pipeline.layout - .validate_push_constant_ranges(stages, offset, end_offset) + set_push_constant(&mut state, stages, offset, size_bytes, values_offset) .map_pass_err(scope)?; - - commands.push(ArcRenderCommand::SetPushConstant { stages, offset, size_bytes, values_offset }); } RenderCommand::Draw { vertex_count, @@ -621,30 +478,16 @@ impl RenderBundleEncoder { let scope = PassErrorScope::Draw { kind: DrawKind::Draw, indexed: false, - pipeline: state.pipeline_id(), }; - let pipeline = state.pipeline(scope)?; - let used_bind_groups = pipeline.used_bind_groups; - - validate_draw( - &state.vertex[..], - &pipeline.steps, - first_vertex, + draw( + &mut state, + &base.dynamic_offsets, vertex_count, - first_instance, instance_count, - ).map_pass_err(scope)?; - - if instance_count > 0 && vertex_count > 0 { - commands.extend(state.flush_vertices()); - commands.extend(state.flush_binds(used_bind_groups, &base.dynamic_offsets)); - commands.push(ArcRenderCommand::Draw { - vertex_count, - instance_count, - first_vertex, - first_instance, - }); - } + first_vertex, + first_instance, + ) + .map_pass_err(scope)?; } RenderCommand::DrawIndexed { index_count, @@ -656,128 +499,45 @@ impl RenderBundleEncoder { let scope = PassErrorScope::Draw { kind: DrawKind::Draw, indexed: true, - pipeline: state.pipeline_id(), - }; - let pipeline = state.pipeline(scope)?; - let used_bind_groups = pipeline.used_bind_groups; - let index = match state.index { - Some(ref index) => index, - None => return Err(DrawError::MissingIndexBuffer).map_pass_err(scope), }; - - validate_indexed_draw( - &state.vertex[..], - &pipeline.steps, - index, - first_index, + draw_indexed( + &mut state, + &base.dynamic_offsets, index_count, - first_instance, instance_count, - ).map_pass_err(scope)?; - - if instance_count > 0 && index_count > 0 { - commands.extend(state.flush_index()); - commands.extend(state.flush_vertices()); - commands.extend(state.flush_binds(used_bind_groups, &base.dynamic_offsets)); - commands.push(ArcRenderCommand::DrawIndexed { index_count, instance_count, first_index, base_vertex, first_instance }); - } + first_index, + base_vertex, + first_instance, + ) + .map_pass_err(scope)?; } RenderCommand::MultiDrawIndirect { buffer_id, offset, count: None, - indexed: false, + indexed, } => { let scope = PassErrorScope::Draw { kind: DrawKind::DrawIndirect, - indexed: false, - pipeline: state.pipeline_id(), + indexed, }; - device - .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) - .map_pass_err(scope)?; - - let pipeline = state.pipeline(scope)?; - let used_bind_groups = pipeline.used_bind_groups; - - let buffer = buffer_guard - .get(buffer_id) - .map_err(|_| RenderCommandError::InvalidBufferId(buffer_id)) - .map_pass_err(scope)?; - - state - .trackers - .buffers.write() - .merge_single(buffer, hal::BufferUses::INDIRECT) - .map_pass_err(scope)?; - - buffer.same_device(device).map_pass_err(scope)?; - buffer.check_usage(wgt::BufferUsages::INDIRECT).map_pass_err(scope)?; - - buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( - buffer, - offset..(offset + mem::size_of::() as u64), - MemoryInitKind::NeedsInitializedMemory, - )); - - commands.extend(state.flush_vertices()); - commands.extend(state.flush_binds(used_bind_groups, &base.dynamic_offsets)); - commands.push(ArcRenderCommand::MultiDrawIndirect { buffer: buffer.clone(), offset, count: None, indexed: false }); - } - RenderCommand::MultiDrawIndirect { - buffer_id, - offset, - count: None, - indexed: true, - } => { - let scope = PassErrorScope::Draw { - kind: DrawKind::DrawIndirect, - indexed: true, - pipeline: state.pipeline_id(), - }; - device - .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) - .map_pass_err(scope)?; - - let pipeline = state.pipeline(scope)?; - let used_bind_groups = pipeline.used_bind_groups; - - let buffer = buffer_guard - .get(buffer_id) - .map_err(|_| RenderCommandError::InvalidBufferId(buffer_id)) - .map_pass_err(scope)?; - - state - .trackers - .buffers.write() - .merge_single(buffer, hal::BufferUses::INDIRECT) - .map_pass_err(scope)?; - - buffer.same_device(device).map_pass_err(scope)?; - buffer.check_usage(wgt::BufferUsages::INDIRECT).map_pass_err(scope)?; - - buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( - buffer, - offset..(offset + mem::size_of::() as u64), - MemoryInitKind::NeedsInitializedMemory, - )); - - let index = match state.index { - Some(ref mut index) => index, - None => return Err(DrawError::MissingIndexBuffer).map_pass_err(scope), - }; - - commands.extend(index.flush()); - commands.extend(state.flush_vertices()); - commands.extend(state.flush_binds(used_bind_groups, &base.dynamic_offsets)); - commands.push(ArcRenderCommand::MultiDrawIndirect { buffer: buffer.clone(), offset, count: None, indexed: true }); + multi_draw_indirect( + &mut state, + &base.dynamic_offsets, + &buffer_guard, + buffer_id, + offset, + indexed, + ) + .map_pass_err(scope)?; } RenderCommand::MultiDrawIndirect { .. } | RenderCommand::MultiDrawIndirectCount { .. } => unimplemented!(), RenderCommand::PushDebugGroup { color: _, len: _ } => unimplemented!(), RenderCommand::InsertDebugMarker { color: _, len: _ } => unimplemented!(), RenderCommand::PopDebugGroup => unimplemented!(), - RenderCommand::WriteTimestamp { .. } // Must check the TIMESTAMP_QUERY_INSIDE_PASSES feature + // Must check the TIMESTAMP_QUERY_INSIDE_PASSES feature + RenderCommand::WriteTimestamp { .. } | RenderCommand::BeginOcclusionQuery { .. } | RenderCommand::EndOcclusionQuery | RenderCommand::BeginPipelineStatisticsQuery { .. } @@ -790,25 +550,38 @@ impl RenderBundleEncoder { } } + let State { + trackers, + flat_dynamic_offsets, + device, + commands, + buffer_memory_init_actions, + texture_memory_init_actions, + .. + } = state; + + let tracker_indices = device.tracker_indices.bundles.clone(); + let discard_hal_labels = device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS); + Ok(RenderBundle { base: BasePass { label: desc.label.as_ref().map(|cow| cow.to_string()), commands, - dynamic_offsets: state.flat_dynamic_offsets, + dynamic_offsets: flat_dynamic_offsets, string_data: Vec::new(), push_constant_data: Vec::new(), }, is_depth_read_only: self.is_depth_read_only, is_stencil_read_only: self.is_stencil_read_only, - device: device.clone(), - used: state.trackers, + device, + used: trackers, buffer_memory_init_actions, texture_memory_init_actions, context: self.context, - info: ResourceInfo::new(&desc.label, Some(device.tracker_indices.bundles.clone())), - discard_hal_labels: device - .instance_flags - .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS), + info: ResourceInfo::new(&desc.label, Some(tracker_indices)), + discard_hal_labels, }) } @@ -828,6 +601,335 @@ impl RenderBundleEncoder { } } +fn set_bind_group( + state: &mut State, + bind_group_guard: &crate::lock::RwLockReadGuard>>, + dynamic_offsets: &[u32], + index: u32, + num_dynamic_offsets: usize, + bind_group_id: id::Id, +) -> Result<(), RenderBundleErrorInner> { + let bind_group = bind_group_guard + .get(bind_group_id) + .map_err(|_| RenderCommandError::InvalidBindGroupId(bind_group_id))?; + + state.trackers.bind_groups.write().add_single(bind_group); + + bind_group.same_device(&state.device)?; + + let max_bind_groups = state.device.limits.max_bind_groups; + if index >= max_bind_groups { + return Err(RenderCommandError::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + } + .into()); + } + + // Identify the next `num_dynamic_offsets` entries from `dynamic_offsets`. + let offsets_range = state.next_dynamic_offset..state.next_dynamic_offset + num_dynamic_offsets; + state.next_dynamic_offset = offsets_range.end; + let offsets = &dynamic_offsets[offsets_range.clone()]; + + bind_group.validate_dynamic_bindings(index, offsets)?; + + state + .buffer_memory_init_actions + .extend_from_slice(&bind_group.used_buffer_ranges); + state + .texture_memory_init_actions + .extend_from_slice(&bind_group.used_texture_ranges); + + state.set_bind_group( + index, + bind_group_guard.get(bind_group_id).as_ref().unwrap(), + &bind_group.layout, + offsets_range, + ); + unsafe { state.trackers.merge_bind_group(&bind_group.used)? }; + // Note: stateless trackers are not merged: the lifetime reference + // is held to the bind group itself. + Ok(()) +} + +fn set_pipeline( + state: &mut State, + pipeline_guard: &crate::lock::RwLockReadGuard>>, + context: &RenderPassContext, + is_depth_read_only: bool, + is_stencil_read_only: bool, + pipeline_id: id::Id, +) -> Result<(), RenderBundleErrorInner> { + let pipeline = pipeline_guard + .get(pipeline_id) + .map_err(|_| RenderCommandError::InvalidPipeline(pipeline_id))?; + + state.trackers.render_pipelines.write().add_single(pipeline); + + pipeline.same_device(&state.device)?; + + context + .check_compatible(&pipeline.pass_context, pipeline.as_ref()) + .map_err(RenderCommandError::IncompatiblePipelineTargets)?; + + if pipeline.flags.contains(PipelineFlags::WRITES_DEPTH) && is_depth_read_only { + return Err(RenderCommandError::IncompatibleDepthAccess(pipeline.error_ident()).into()); + } + if pipeline.flags.contains(PipelineFlags::WRITES_STENCIL) && is_stencil_read_only { + return Err(RenderCommandError::IncompatibleStencilAccess(pipeline.error_ident()).into()); + } + + let pipeline_state = PipelineState::new(pipeline); + + state + .commands + .push(ArcRenderCommand::SetPipeline(pipeline.clone())); + + // If this pipeline uses push constants, zero out their values. + if let Some(iter) = pipeline_state.zero_push_constants() { + state.commands.extend(iter) + } + + state.invalidate_bind_groups(&pipeline_state, &pipeline.layout); + state.pipeline = Some(pipeline_state); + Ok(()) +} + +fn set_index_buffer( + state: &mut State, + buffer_guard: &crate::lock::RwLockReadGuard>>, + buffer_id: id::Id, + index_format: wgt::IndexFormat, + offset: u64, + size: Option, +) -> Result<(), RenderBundleErrorInner> { + let buffer = buffer_guard + .get(buffer_id) + .map_err(|_| RenderCommandError::InvalidBufferId(buffer_id))?; + + state + .trackers + .buffers + .write() + .merge_single(buffer, hal::BufferUses::INDEX)?; + + buffer.same_device(&state.device)?; + buffer.check_usage(wgt::BufferUsages::INDEX)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + buffer, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + )); + state.set_index_buffer(buffer.clone(), index_format, offset..end); + Ok(()) +} + +fn set_vertex_buffer( + state: &mut State, + buffer_guard: &crate::lock::RwLockReadGuard>>, + slot: u32, + buffer_id: id::Id, + offset: u64, + size: Option, +) -> Result<(), RenderBundleErrorInner> { + let max_vertex_buffers = state.device.limits.max_vertex_buffers; + if slot >= max_vertex_buffers { + return Err(RenderCommandError::VertexBufferIndexOutOfRange { + index: slot, + max: max_vertex_buffers, + } + .into()); + } + + let buffer = buffer_guard + .get(buffer_id) + .map_err(|_| RenderCommandError::InvalidBufferId(buffer_id))?; + + state + .trackers + .buffers + .write() + .merge_single(buffer, hal::BufferUses::VERTEX)?; + + buffer.same_device(&state.device)?; + buffer.check_usage(wgt::BufferUsages::VERTEX)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + buffer, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + )); + state.vertex[slot as usize] = Some(VertexState::new(buffer.clone(), offset..end)); + Ok(()) +} + +fn set_push_constant( + state: &mut State, + stages: wgt::ShaderStages, + offset: u32, + size_bytes: u32, + values_offset: Option, +) -> Result<(), RenderBundleErrorInner> { + let end_offset = offset + size_bytes; + + let pipeline_state = state.pipeline()?; + + pipeline_state + .pipeline + .layout + .validate_push_constant_ranges(stages, offset, end_offset)?; + + state.commands.push(ArcRenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset, + }); + Ok(()) +} + +fn draw( + state: &mut State, + dynamic_offsets: &[u32], + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, +) -> Result<(), RenderBundleErrorInner> { + let pipeline = state.pipeline()?; + let used_bind_groups = pipeline.used_bind_groups; + + validate_draw( + &state.vertex[..], + &pipeline.steps, + first_vertex, + vertex_count, + first_instance, + instance_count, + )?; + + if instance_count > 0 && vertex_count > 0 { + state.flush_vertices(); + state.flush_binds(used_bind_groups, dynamic_offsets); + state.commands.push(ArcRenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + }); + } + Ok(()) +} + +fn draw_indexed( + state: &mut State, + dynamic_offsets: &[u32], + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, +) -> Result<(), RenderBundleErrorInner> { + let pipeline = state.pipeline()?; + let used_bind_groups = pipeline.used_bind_groups; + let index = match state.index { + Some(ref index) => index, + None => return Err(DrawError::MissingIndexBuffer.into()), + }; + + validate_indexed_draw( + &state.vertex[..], + &pipeline.steps, + index, + first_index, + index_count, + first_instance, + instance_count, + )?; + + if instance_count > 0 && index_count > 0 { + state.flush_index(); + state.flush_vertices(); + state.flush_binds(used_bind_groups, dynamic_offsets); + state.commands.push(ArcRenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + }); + } + Ok(()) +} + +fn multi_draw_indirect( + state: &mut State, + dynamic_offsets: &[u32], + buffer_guard: &crate::lock::RwLockReadGuard>>, + buffer_id: id::Id, + offset: u64, + indexed: bool, +) -> Result<(), RenderBundleErrorInner> { + state + .device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?; + + let pipeline = state.pipeline()?; + let used_bind_groups = pipeline.used_bind_groups; + + let buffer = buffer_guard + .get(buffer_id) + .map_err(|_| RenderCommandError::InvalidBufferId(buffer_id))?; + + state + .trackers + .buffers + .write() + .merge_single(buffer, hal::BufferUses::INDIRECT)?; + + buffer.same_device(&state.device)?; + buffer.check_usage(wgt::BufferUsages::INDIRECT)?; + + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + buffer, + offset..(offset + mem::size_of::() as u64), + MemoryInitKind::NeedsInitializedMemory, + )); + + if indexed { + let index = match state.index { + Some(ref mut index) => index, + None => return Err(DrawError::MissingIndexBuffer.into()), + }; + state.commands.extend(index.flush()); + } + + state.flush_vertices(); + state.flush_binds(used_bind_groups, dynamic_offsets); + state.commands.push(ArcRenderCommand::MultiDrawIndirect { + buffer: buffer.clone(), + offset, + count: None, + indexed, + }); + Ok(()) +} + /// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid. #[derive(Clone, Debug, Error)] #[non_exhaustive] @@ -1209,8 +1311,6 @@ struct PipelineState { /// The pipeline pipeline: Arc>, - pipeline_id: id::RenderPipelineId, - /// How this pipeline's vertex shader traverses each vertex buffer, indexed /// by vertex buffer slot number. steps: Vec, @@ -1224,10 +1324,9 @@ struct PipelineState { } impl PipelineState { - fn new(pipeline: &Arc>, pipeline_id: id::RenderPipelineId) -> Self { + fn new(pipeline: &Arc>) -> Self { Self { pipeline: pipeline.clone(), - pipeline_id, steps: pipeline.vertex_steps.to_vec(), push_constant_ranges: pipeline .layout @@ -1296,20 +1395,20 @@ struct State { /// /// [`dynamic_offsets`]: BasePass::dynamic_offsets flat_dynamic_offsets: Vec, + + device: Arc>, + commands: Vec>, + buffer_memory_init_actions: Vec>, + texture_memory_init_actions: Vec>, + next_dynamic_offset: usize, } impl State { - /// Return the id of the current pipeline, if any. - fn pipeline_id(&self) -> Option { - self.pipeline.as_ref().map(|p| p.pipeline_id) - } - /// Return the current pipeline state. Return an error if none is set. - fn pipeline(&self, scope: PassErrorScope) -> Result<&PipelineState, RenderBundleError> { + fn pipeline(&self) -> Result<&PipelineState, RenderBundleErrorInner> { self.pipeline .as_ref() - .ok_or(DrawError::MissingPipeline) - .map_pass_err(scope) + .ok_or(DrawError::MissingPipeline.into()) } /// Mark all non-empty bind group table entries from `index` onwards as dirty. @@ -1422,23 +1521,22 @@ impl State { /// Generate a `SetIndexBuffer` command to prepare for an indexed draw /// command, if needed. - fn flush_index(&mut self) -> Option> { - self.index.as_mut().and_then(|index| index.flush()) + fn flush_index(&mut self) { + let commands = self.index.as_mut().and_then(|index| index.flush()); + self.commands.extend(commands); } - fn flush_vertices(&mut self) -> impl Iterator> + '_ { - self.vertex + fn flush_vertices(&mut self) { + let commands = self + .vertex .iter_mut() .enumerate() - .flat_map(|(i, vs)| vs.as_mut().and_then(|vs| vs.flush(i as u32))) + .flat_map(|(i, vs)| vs.as_mut().and_then(|vs| vs.flush(i as u32))); + self.commands.extend(commands); } /// Generate `SetBindGroup` commands for any bind groups that need to be updated. - fn flush_binds( - &mut self, - used_bind_groups: usize, - dynamic_offsets: &[wgt::DynamicOffset], - ) -> impl Iterator> + '_ { + fn flush_binds(&mut self, used_bind_groups: usize, dynamic_offsets: &[wgt::DynamicOffset]) { // Append each dirty bind group's dynamic offsets to `flat_dynamic_offsets`. for contents in self.bind[..used_bind_groups].iter().flatten() { if contents.is_dirty { @@ -1449,7 +1547,7 @@ impl State { // Then, generate `SetBindGroup` commands to update the dirty bind // groups. After this, all bind groups are clean. - self.bind[..used_bind_groups] + let commands = self.bind[..used_bind_groups] .iter_mut() .enumerate() .flat_map(|(i, entry)| { @@ -1465,7 +1563,9 @@ impl State { } } None - }) + }); + + self.commands.extend(commands); } } @@ -1480,6 +1580,8 @@ pub(super) enum RenderBundleErrorInner { Draw(#[from] DrawError), #[error(transparent)] MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error(transparent)] + Bind(#[from] BindError), } impl From for RenderBundleErrorInner diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index e4831bdb0e..73b8838073 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -1,5 +1,7 @@ use crate::{ - binding_model::{BindError, LateMinBufferBindingSizeMismatch, PushConstantUploadError}, + binding_model::{ + BindError, BindGroup, LateMinBufferBindingSizeMismatch, PushConstantUploadError, + }, command::{ bind::Binder, compute_command::{ArcComputeCommand, ComputeCommand}, @@ -9,13 +11,17 @@ use crate::{ CommandBuffer, CommandEncoderError, CommandEncoderStatus, MapPassErr, PassErrorScope, QueryUseError, StateChange, }, - device::{DeviceError, MissingDownlevelFlags, MissingFeatures}, + device::{Device, DeviceError, MissingDownlevelFlags, MissingFeatures}, error::{ErrorFormatter, PrettyError}, global::Global, hal_api::HalApi, hal_label, id, - init_tracker::MemoryInitKind, - resource::{self, DestroyedResourceError, MissingBufferUsageError, ParentDevice, Resource}, + init_tracker::{BufferInitTrackerAction, MemoryInitKind}, + pipeline::ComputePipeline, + resource::{ + self, Buffer, DestroyedResourceError, MissingBufferUsageError, ParentDevice, Resource, + ResourceErrorIdent, + }, snatch::SnatchGuard, track::{ResourceUsageCompatibilityError, Tracker, TrackerIndex, UsageScope}, Label, @@ -33,7 +39,7 @@ use wgt::{BufferAddress, DynamicOffset}; use std::sync::Arc; use std::{fmt, mem, str}; -use super::DynComputePass; +use super::{memory_init::CommandBufferTextureMemoryActions, DynComputePass}; pub struct ComputePass { /// All pass data & records is stored here. @@ -132,13 +138,17 @@ struct ArcComputePassDescriptor<'a, A: HalApi> { pub timestamp_writes: Option>, } -#[derive(Clone, Debug, Error, Eq, PartialEq)] +#[derive(Clone, Debug, Error)] #[non_exhaustive] pub enum DispatchError { #[error("Compute pipeline must be set")] MissingPipeline, - #[error("Incompatible bind group at index {index} in the current compute pipeline")] - IncompatibleBindGroup { index: u32, diff: Vec }, + #[error("Bind group at index {index} is incompatible with the current set {pipeline}")] + IncompatibleBindGroup { + index: u32, + pipeline: ResourceErrorIdent, + diff: Vec, + }, #[error( "Each current dispatch group size dimension ({current:?}) must be less or equal to {limit}" )] @@ -248,41 +258,59 @@ where } } -struct State<'a, A: HalApi> { +struct State<'scope, 'snatch_guard, 'cmd_buf, 'raw_encoder, A: HalApi> { binder: Binder, - pipeline: Option, - scope: UsageScope<'a, A>, + pipeline: Option>>, + scope: UsageScope<'scope, A>, debug_scope_depth: u32, + + snatch_guard: SnatchGuard<'snatch_guard>, + + device: &'cmd_buf Arc>, + + raw_encoder: &'raw_encoder mut A::CommandEncoder, + + tracker: &'cmd_buf mut Tracker, + buffer_memory_init_actions: &'cmd_buf mut Vec>, + texture_memory_actions: &'cmd_buf mut CommandBufferTextureMemoryActions, + + temp_offsets: Vec, + dynamic_offset_count: usize, + string_offset: usize, + active_query: Option<(Arc>, u32)>, + + intermediate_trackers: Tracker, + + /// Immediate texture inits required because of prior discards. Need to + /// be inserted before texture reads. + pending_discard_init_fixups: SurfacesInDiscardState, } -impl<'a, A: HalApi> State<'a, A> { +impl<'scope, 'snatch_guard, 'cmd_buf, 'raw_encoder, A: HalApi> + State<'scope, 'snatch_guard, 'cmd_buf, 'raw_encoder, A> +{ fn is_ready(&self) -> Result<(), DispatchError> { - let bind_mask = self.binder.invalid_mask(); - if bind_mask != 0 { - //let (expected, provided) = self.binder.entries[index as usize].info(); - let index = bind_mask.trailing_zeros(); - - return Err(DispatchError::IncompatibleBindGroup { - index, - diff: self.binder.bgl_diff(), - }); - } - if self.pipeline.is_none() { - return Err(DispatchError::MissingPipeline); + if let Some(pipeline) = self.pipeline.as_ref() { + let bind_mask = self.binder.invalid_mask(); + if bind_mask != 0 { + return Err(DispatchError::IncompatibleBindGroup { + index: bind_mask.trailing_zeros(), + pipeline: pipeline.error_ident(), + diff: self.binder.bgl_diff(), + }); + } + self.binder.check_late_buffer_bindings()?; + Ok(()) + } else { + Err(DispatchError::MissingPipeline) } - self.binder.check_late_buffer_bindings()?; - - Ok(()) } // `extra_buffer` is there to represent the indirect buffer that is also // part of the usage scope. fn flush_states( &mut self, - raw_encoder: &mut A::CommandEncoder, - base_trackers: &mut Tracker, indirect_buffer: Option, - snatch_guard: &SnatchGuard, ) -> Result<(), ResourceUsageCompatibilityError> { for bind_group in self.binder.list_active() { unsafe { self.scope.merge_bind_group(&bind_group.used)? }; @@ -292,21 +320,25 @@ impl<'a, A: HalApi> State<'a, A> { for bind_group in self.binder.list_active() { unsafe { - base_trackers + self.intermediate_trackers .set_and_remove_from_usage_scope_sparse(&mut self.scope, &bind_group.used) } } // Add the state of the indirect buffer if it hasn't been hit before. unsafe { - base_trackers + self.intermediate_trackers .buffers .set_and_remove_from_usage_scope_sparse(&mut self.scope.buffers, indirect_buffer); } log::trace!("Encoding dispatch barriers"); - CommandBuffer::drain_barriers(raw_encoder, base_trackers, snatch_guard); + CommandBuffer::drain_barriers( + self.raw_encoder, + &mut self.intermediate_trackers, + &self.snatch_guard, + ); Ok(()) } } @@ -474,9 +506,6 @@ impl Global { let encoder = &mut cmd_buf_data.encoder; let status = &mut cmd_buf_data.status; - let tracker = &mut cmd_buf_data.trackers; - let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions; - let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions; // We automatically keep extending command buffers over time, and because // we want to insert a command buffer _before_ what we're about to record, @@ -484,32 +513,47 @@ impl Global { encoder.close().map_pass_err(pass_scope)?; // will be reset to true if recording is done without errors *status = CommandEncoderStatus::Error; - let raw = encoder.open().map_pass_err(pass_scope)?; + let raw_encoder = encoder.open().map_pass_err(pass_scope)?; let mut state = State { binder: Binder::new(), pipeline: None, scope: device.new_usage_scope(), debug_scope_depth: 0, + + snatch_guard: device.snatchable_lock.read(), + + device, + raw_encoder, + tracker: &mut cmd_buf_data.trackers, + buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions, + texture_memory_actions: &mut cmd_buf_data.texture_memory_actions, + + temp_offsets: Vec::new(), + dynamic_offset_count: 0, + string_offset: 0, + active_query: None, + + intermediate_trackers: Tracker::new(), + + pending_discard_init_fixups: SurfacesInDiscardState::new(), }; - let mut temp_offsets = Vec::new(); - let mut dynamic_offset_count = 0; - let mut string_offset = 0; - let mut active_query = None; - - let snatch_guard = device.snatchable_lock.read(); - - let indices = &device.tracker_indices; - tracker.buffers.set_size(indices.buffers.size()); - tracker.textures.set_size(indices.textures.size()); - tracker.bind_groups.set_size(indices.bind_groups.size()); - tracker + + let indices = &state.device.tracker_indices; + state.tracker.buffers.set_size(indices.buffers.size()); + state.tracker.textures.set_size(indices.textures.size()); + state + .tracker + .bind_groups + .set_size(indices.bind_groups.size()); + state + .tracker .compute_pipelines .set_size(indices.compute_pipelines.size()); - tracker.query_sets.set_size(indices.query_sets.size()); + state.tracker.query_sets.set_size(indices.query_sets.size()); let timestamp_writes = if let Some(tw) = timestamp_writes.take() { - let query_set = tracker.query_sets.insert_single(tw.query_set); + let query_set = state.tracker.query_sets.insert_single(tw.query_set); // Unlike in render passes we can't delay resetting the query sets since // there is no auxiliary pass. @@ -526,7 +570,9 @@ impl Global { // But no point in erroring over that nuance here! if let Some(range) = range { unsafe { - raw.reset_queries(query_set.raw.as_ref().unwrap(), range); + state + .raw_encoder + .reset_queries(query_set.raw.as_ref().unwrap(), range); } } @@ -539,25 +585,15 @@ impl Global { None }; - let discard_hal_labels = self - .instance - .flags - .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS); let hal_desc = hal::ComputePassDescriptor { label: hal_label(base.label.as_deref(), self.instance.flags), timestamp_writes, }; unsafe { - raw.begin_compute_pass(&hal_desc); + state.raw_encoder.begin_compute_pass(&hal_desc); } - let mut intermediate_trackers = Tracker::::new(); - - // Immediate texture inits required because of prior discards. Need to - // be inserted before texture reads. - let mut pending_discard_init_fixups = SurfacesInDiscardState::new(); - // TODO: We should be draining the commands here, avoiding extra copies in the process. // (A command encoder can't be executed twice!) for command in base.commands { @@ -567,133 +603,20 @@ impl Global { num_dynamic_offsets, bind_group, } => { - let scope = PassErrorScope::SetBindGroup(bind_group.as_info().id()); - - bind_group.same_device_as(cmd_buf).map_pass_err(scope)?; - - let max_bind_groups = cmd_buf.limits.max_bind_groups; - if index >= max_bind_groups { - return Err(ComputePassErrorInner::BindGroupIndexOutOfRange { - index, - max: max_bind_groups, - }) - .map_pass_err(scope); - } - - temp_offsets.clear(); - temp_offsets.extend_from_slice( - &base.dynamic_offsets - [dynamic_offset_count..dynamic_offset_count + num_dynamic_offsets], - ); - dynamic_offset_count += num_dynamic_offsets; - - let bind_group = tracker.bind_groups.insert_single(bind_group); - bind_group - .validate_dynamic_bindings(index, &temp_offsets, &cmd_buf.limits) - .map_pass_err(scope)?; - - buffer_memory_init_actions.extend( - bind_group.used_buffer_ranges.iter().filter_map(|action| { - action - .buffer - .initialization_status - .read() - .check_action(action) - }), - ); - - for action in bind_group.used_texture_ranges.iter() { - pending_discard_init_fixups - .extend(texture_memory_actions.register_init_action(action)); - } - - let pipeline_layout = state.binder.pipeline_layout.clone(); - let entries = - state - .binder - .assign_group(index as usize, bind_group, &temp_offsets); - if !entries.is_empty() && pipeline_layout.is_some() { - let pipeline_layout = pipeline_layout.as_ref().unwrap().raw(); - for (i, e) in entries.iter().enumerate() { - if let Some(group) = e.group.as_ref() { - let raw_bg = group.try_raw(&snatch_guard).map_pass_err(scope)?; - unsafe { - raw.set_bind_group( - pipeline_layout, - index + i as u32, - raw_bg, - &e.dynamic_offsets, - ); - } - } - } - } + let scope = PassErrorScope::SetBindGroup; + set_bind_group( + &mut state, + cmd_buf, + &base.dynamic_offsets, + index, + num_dynamic_offsets, + bind_group, + ) + .map_pass_err(scope)?; } ArcComputeCommand::SetPipeline(pipeline) => { - let pipeline_id = pipeline.as_info().id(); - let scope = PassErrorScope::SetPipelineCompute(pipeline_id); - - pipeline.same_device_as(cmd_buf).map_pass_err(scope)?; - - state.pipeline = Some(pipeline_id); - - let pipeline = tracker.compute_pipelines.insert_single(pipeline); - - unsafe { - raw.set_compute_pipeline(pipeline.raw()); - } - - // Rebind resources - if state.binder.pipeline_layout.is_none() - || !state - .binder - .pipeline_layout - .as_ref() - .unwrap() - .is_equal(&pipeline.layout) - { - let (start_index, entries) = state.binder.change_pipeline_layout( - &pipeline.layout, - &pipeline.late_sized_buffer_groups, - ); - if !entries.is_empty() { - for (i, e) in entries.iter().enumerate() { - if let Some(group) = e.group.as_ref() { - let raw_bg = - group.try_raw(&snatch_guard).map_pass_err(scope)?; - unsafe { - raw.set_bind_group( - pipeline.layout.raw(), - start_index as u32 + i as u32, - raw_bg, - &e.dynamic_offsets, - ); - } - } - } - } - - // Clear push constant ranges - let non_overlapping = super::bind::compute_nonoverlapping_ranges( - &pipeline.layout.push_constant_ranges, - ); - for range in non_overlapping { - let offset = range.range.start; - let size_bytes = range.range.end - offset; - super::push_constant_clear( - offset, - size_bytes, - |clear_offset, clear_data| unsafe { - raw.set_push_constants( - pipeline.layout.raw(), - wgt::ShaderStages::COMPUTE, - clear_offset, - clear_data, - ); - }, - ); - } - } + let scope = PassErrorScope::SetPipelineCompute; + set_pipeline(&mut state, cmd_buf, pipeline).map_pass_err(scope)?; } ArcComputeCommand::SetPushConstant { offset, @@ -701,178 +624,39 @@ impl Global { values_offset, } => { let scope = PassErrorScope::SetPushConstant; - - let end_offset_bytes = offset + size_bytes; - let values_end_offset = - (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; - let data_slice = - &base.push_constant_data[(values_offset as usize)..values_end_offset]; - - let pipeline_layout = state - .binder - .pipeline_layout - .as_ref() - //TODO: don't error here, lazily update the push constants - .ok_or(ComputePassErrorInner::Dispatch( - DispatchError::MissingPipeline, - )) - .map_pass_err(scope)?; - - pipeline_layout - .validate_push_constant_ranges( - wgt::ShaderStages::COMPUTE, - offset, - end_offset_bytes, - ) - .map_pass_err(scope)?; - - unsafe { - raw.set_push_constants( - pipeline_layout.raw(), - wgt::ShaderStages::COMPUTE, - offset, - data_slice, - ); - } + set_push_constant( + &mut state, + &base.push_constant_data, + offset, + size_bytes, + values_offset, + ) + .map_pass_err(scope)?; } ArcComputeCommand::Dispatch(groups) => { - let scope = PassErrorScope::Dispatch { - indirect: false, - pipeline: state.pipeline, - }; - state.is_ready().map_pass_err(scope)?; - - state - .flush_states(raw, &mut intermediate_trackers, None, &snatch_guard) - .map_pass_err(scope)?; - - let groups_size_limit = cmd_buf.limits.max_compute_workgroups_per_dimension; - - if groups[0] > groups_size_limit - || groups[1] > groups_size_limit - || groups[2] > groups_size_limit - { - return Err(ComputePassErrorInner::Dispatch( - DispatchError::InvalidGroupSize { - current: groups, - limit: groups_size_limit, - }, - )) - .map_pass_err(scope); - } - - unsafe { - raw.dispatch(groups); - } + let scope = PassErrorScope::Dispatch { indirect: false }; + dispatch(&mut state, groups).map_pass_err(scope)?; } ArcComputeCommand::DispatchIndirect { buffer, offset } => { - let scope = PassErrorScope::Dispatch { - indirect: true, - pipeline: state.pipeline, - }; - - buffer.same_device_as(cmd_buf).map_pass_err(scope)?; - - state.is_ready().map_pass_err(scope)?; - - device - .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) - .map_pass_err(scope)?; - - state - .scope - .buffers - .merge_single(&buffer, hal::BufferUses::INDIRECT) - .map_pass_err(scope)?; - buffer - .check_usage(wgt::BufferUsages::INDIRECT) - .map_pass_err(scope)?; - - let end_offset = offset + mem::size_of::() as u64; - if end_offset > buffer.size { - return Err(ComputePassErrorInner::IndirectBufferOverrun { - offset, - end_offset, - buffer_size: buffer.size, - }) - .map_pass_err(scope); - } - - let buf_raw = buffer.try_raw(&snatch_guard).map_pass_err(scope)?; - - let stride = 3 * 4; // 3 integers, x/y/z group size - - buffer_memory_init_actions.extend( - buffer.initialization_status.read().create_action( - &buffer, - offset..(offset + stride), - MemoryInitKind::NeedsInitializedMemory, - ), - ); - - state - .flush_states( - raw, - &mut intermediate_trackers, - Some(buffer.as_info().tracker_index()), - &snatch_guard, - ) - .map_pass_err(scope)?; - unsafe { - raw.dispatch_indirect(buf_raw, offset); - } + let scope = PassErrorScope::Dispatch { indirect: true }; + dispatch_indirect(&mut state, cmd_buf, buffer, offset).map_pass_err(scope)?; } ArcComputeCommand::PushDebugGroup { color: _, len } => { - state.debug_scope_depth += 1; - if !discard_hal_labels { - let label = - str::from_utf8(&base.string_data[string_offset..string_offset + len]) - .unwrap(); - unsafe { - raw.begin_debug_marker(label); - } - } - string_offset += len; + push_debug_group(&mut state, &base.string_data, len); } ArcComputeCommand::PopDebugGroup => { let scope = PassErrorScope::PopDebugGroup; - - if state.debug_scope_depth == 0 { - return Err(ComputePassErrorInner::InvalidPopDebugGroup) - .map_pass_err(scope); - } - state.debug_scope_depth -= 1; - if !discard_hal_labels { - unsafe { - raw.end_debug_marker(); - } - } + pop_debug_group(&mut state).map_pass_err(scope)?; } ArcComputeCommand::InsertDebugMarker { color: _, len } => { - if !discard_hal_labels { - let label = - str::from_utf8(&base.string_data[string_offset..string_offset + len]) - .unwrap(); - unsafe { raw.insert_debug_marker(label) } - } - string_offset += len; + insert_debug_marker(&mut state, &base.string_data, len); } ArcComputeCommand::WriteTimestamp { query_set, query_index, } => { let scope = PassErrorScope::WriteTimestamp; - - query_set.same_device_as(cmd_buf).map_pass_err(scope)?; - - device - .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES) - .map_pass_err(scope)?; - - let query_set = tracker.query_sets.insert_single(query_set); - - query_set - .validate_and_write_timestamp(raw, query_index, None) + write_timestamp(&mut state, cmd_buf, query_set, query_index) .map_pass_err(scope)?; } ArcComputeCommand::BeginPipelineStatisticsQuery { @@ -880,35 +664,41 @@ impl Global { query_index, } => { let scope = PassErrorScope::BeginPipelineStatisticsQuery; - - query_set.same_device_as(cmd_buf).map_pass_err(scope)?; - - let query_set = tracker.query_sets.insert_single(query_set); - validate_and_begin_pipeline_statistics_query( - query_set.clone(), - raw, + query_set, + state.raw_encoder, + &mut state.tracker.query_sets, + cmd_buf, query_index, None, - &mut active_query, + &mut state.active_query, ) .map_pass_err(scope)?; } ArcComputeCommand::EndPipelineStatisticsQuery => { let scope = PassErrorScope::EndPipelineStatisticsQuery; - end_pipeline_statistics_query(raw, &mut active_query).map_pass_err(scope)?; + end_pipeline_statistics_query(state.raw_encoder, &mut state.active_query) + .map_pass_err(scope)?; } } } unsafe { - raw.end_compute_pass(); + state.raw_encoder.end_compute_pass(); } // We've successfully recorded the compute pass, bring the // command buffer out of the error state. *status = CommandEncoderStatus::Recording; + let State { + snatch_guard, + tracker, + intermediate_trackers, + pending_discard_init_fixups, + .. + } = state; + // Stop the current command buffer. encoder.close().map_pass_err(pass_scope)?; @@ -936,6 +726,312 @@ impl Global { } } +fn set_bind_group( + state: &mut State, + cmd_buf: &CommandBuffer, + dynamic_offsets: &[DynamicOffset], + index: u32, + num_dynamic_offsets: usize, + bind_group: Arc>, +) -> Result<(), ComputePassErrorInner> { + bind_group.same_device_as(cmd_buf)?; + + let max_bind_groups = state.device.limits.max_bind_groups; + if index >= max_bind_groups { + return Err(ComputePassErrorInner::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }); + } + + state.temp_offsets.clear(); + state.temp_offsets.extend_from_slice( + &dynamic_offsets + [state.dynamic_offset_count..state.dynamic_offset_count + num_dynamic_offsets], + ); + state.dynamic_offset_count += num_dynamic_offsets; + + let bind_group = state.tracker.bind_groups.insert_single(bind_group); + bind_group.validate_dynamic_bindings(index, &state.temp_offsets)?; + + state + .buffer_memory_init_actions + .extend(bind_group.used_buffer_ranges.iter().filter_map(|action| { + action + .buffer + .initialization_status + .read() + .check_action(action) + })); + + for action in bind_group.used_texture_ranges.iter() { + state + .pending_discard_init_fixups + .extend(state.texture_memory_actions.register_init_action(action)); + } + + let pipeline_layout = state.binder.pipeline_layout.clone(); + let entries = state + .binder + .assign_group(index as usize, bind_group, &state.temp_offsets); + if !entries.is_empty() && pipeline_layout.is_some() { + let pipeline_layout = pipeline_layout.as_ref().unwrap().raw(); + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group.try_raw(&state.snatch_guard)?; + unsafe { + state.raw_encoder.set_bind_group( + pipeline_layout, + index + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + Ok(()) +} + +fn set_pipeline( + state: &mut State, + cmd_buf: &CommandBuffer, + pipeline: Arc>, +) -> Result<(), ComputePassErrorInner> { + pipeline.same_device_as(cmd_buf)?; + + state.pipeline = Some(pipeline.clone()); + + let pipeline = state.tracker.compute_pipelines.insert_single(pipeline); + + unsafe { + state.raw_encoder.set_compute_pipeline(pipeline.raw()); + } + + // Rebind resources + if state.binder.pipeline_layout.is_none() + || !state + .binder + .pipeline_layout + .as_ref() + .unwrap() + .is_equal(&pipeline.layout) + { + let (start_index, entries) = state + .binder + .change_pipeline_layout(&pipeline.layout, &pipeline.late_sized_buffer_groups); + if !entries.is_empty() { + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group.try_raw(&state.snatch_guard)?; + unsafe { + state.raw_encoder.set_bind_group( + pipeline.layout.raw(), + start_index as u32 + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + + // Clear push constant ranges + let non_overlapping = + super::bind::compute_nonoverlapping_ranges(&pipeline.layout.push_constant_ranges); + for range in non_overlapping { + let offset = range.range.start; + let size_bytes = range.range.end - offset; + super::push_constant_clear(offset, size_bytes, |clear_offset, clear_data| unsafe { + state.raw_encoder.set_push_constants( + pipeline.layout.raw(), + wgt::ShaderStages::COMPUTE, + clear_offset, + clear_data, + ); + }); + } + } + Ok(()) +} + +fn set_push_constant( + state: &mut State, + push_constant_data: &[u32], + offset: u32, + size_bytes: u32, + values_offset: u32, +) -> Result<(), ComputePassErrorInner> { + let end_offset_bytes = offset + size_bytes; + let values_end_offset = (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = &push_constant_data[(values_offset as usize)..values_end_offset]; + + let pipeline_layout = state + .binder + .pipeline_layout + .as_ref() + //TODO: don't error here, lazily update the push constants + .ok_or(ComputePassErrorInner::Dispatch( + DispatchError::MissingPipeline, + ))?; + + pipeline_layout.validate_push_constant_ranges( + wgt::ShaderStages::COMPUTE, + offset, + end_offset_bytes, + )?; + + unsafe { + state.raw_encoder.set_push_constants( + pipeline_layout.raw(), + wgt::ShaderStages::COMPUTE, + offset, + data_slice, + ); + } + Ok(()) +} + +fn dispatch( + state: &mut State, + groups: [u32; 3], +) -> Result<(), ComputePassErrorInner> { + state.is_ready()?; + + state.flush_states(None)?; + + let groups_size_limit = state.device.limits.max_compute_workgroups_per_dimension; + + if groups[0] > groups_size_limit + || groups[1] > groups_size_limit + || groups[2] > groups_size_limit + { + return Err(ComputePassErrorInner::Dispatch( + DispatchError::InvalidGroupSize { + current: groups, + limit: groups_size_limit, + }, + )); + } + + unsafe { + state.raw_encoder.dispatch(groups); + } + Ok(()) +} + +fn dispatch_indirect( + state: &mut State, + cmd_buf: &CommandBuffer, + buffer: Arc>, + offset: u64, +) -> Result<(), ComputePassErrorInner> { + buffer.same_device_as(cmd_buf)?; + + state.is_ready()?; + + state + .device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?; + + state + .scope + .buffers + .merge_single(&buffer, hal::BufferUses::INDIRECT)?; + buffer.check_usage(wgt::BufferUsages::INDIRECT)?; + + let end_offset = offset + mem::size_of::() as u64; + if end_offset > buffer.size { + return Err(ComputePassErrorInner::IndirectBufferOverrun { + offset, + end_offset, + buffer_size: buffer.size, + }); + } + + let stride = 3 * 4; // 3 integers, x/y/z group size + + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + &buffer, + offset..(offset + stride), + MemoryInitKind::NeedsInitializedMemory, + )); + + state.flush_states(Some(buffer.as_info().tracker_index()))?; + + let buf_raw = buffer.try_raw(&state.snatch_guard)?; + unsafe { + state.raw_encoder.dispatch_indirect(buf_raw, offset); + } + Ok(()) +} + +fn push_debug_group(state: &mut State, string_data: &[u8], len: usize) { + state.debug_scope_depth += 1; + if !state + .device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + let label = + str::from_utf8(&string_data[state.string_offset..state.string_offset + len]).unwrap(); + unsafe { + state.raw_encoder.begin_debug_marker(label); + } + } + state.string_offset += len; +} + +fn pop_debug_group(state: &mut State) -> Result<(), ComputePassErrorInner> { + if state.debug_scope_depth == 0 { + return Err(ComputePassErrorInner::InvalidPopDebugGroup); + } + state.debug_scope_depth -= 1; + if !state + .device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + unsafe { + state.raw_encoder.end_debug_marker(); + } + } + Ok(()) +} + +fn insert_debug_marker(state: &mut State, string_data: &[u8], len: usize) { + if !state + .device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + let label = + str::from_utf8(&string_data[state.string_offset..state.string_offset + len]).unwrap(); + unsafe { state.raw_encoder.insert_debug_marker(label) } + } + state.string_offset += len; +} + +fn write_timestamp( + state: &mut State, + cmd_buf: &CommandBuffer, + query_set: Arc>, + query_index: u32, +) -> Result<(), ComputePassErrorInner> { + query_set.same_device_as(cmd_buf)?; + + state + .device + .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES)?; + + let query_set = state.tracker.query_sets.insert_single(query_set); + + query_set.validate_and_write_timestamp(state.raw_encoder, query_index, None)?; + Ok(()) +} + // Recording a compute pass. impl Global { pub fn compute_pass_set_bind_group( @@ -945,7 +1041,7 @@ impl Global { bind_group_id: id::BindGroupId, offsets: &[DynamicOffset], ) -> Result<(), ComputePassError> { - let scope = PassErrorScope::SetBindGroup(bind_group_id); + let scope = PassErrorScope::SetBindGroup; let base = pass .base .as_mut() @@ -987,7 +1083,7 @@ impl Global { ) -> Result<(), ComputePassError> { let redundant = pass.current_pipeline.set_and_check_redundant(pipeline_id); - let scope = PassErrorScope::SetPipelineCompute(pipeline_id); + let scope = PassErrorScope::SetPipelineCompute; let base = pass.base_mut(scope)?; if redundant { @@ -1052,10 +1148,7 @@ impl Global { groups_y: u32, groups_z: u32, ) -> Result<(), ComputePassError> { - let scope = PassErrorScope::Dispatch { - indirect: false, - pipeline: pass.current_pipeline.last_state, - }; + let scope = PassErrorScope::Dispatch { indirect: false }; let base = pass.base_mut(scope)?; base.commands.push(ArcComputeCommand::::Dispatch([ @@ -1072,10 +1165,7 @@ impl Global { offset: BufferAddress, ) -> Result<(), ComputePassError> { let hub = A::hub(self); - let scope = PassErrorScope::Dispatch { - indirect: true, - pipeline: pass.current_pipeline.last_state, - }; + let scope = PassErrorScope::Dispatch { indirect: true }; let base = pass.base_mut(scope)?; let buffer = hub diff --git a/wgpu-core/src/command/compute_command.rs b/wgpu-core/src/command/compute_command.rs index bd98bda157..eb8ce9fa34 100644 --- a/wgpu-core/src/command/compute_command.rs +++ b/wgpu-core/src/command/compute_command.rs @@ -97,7 +97,7 @@ impl ComputeCommand { num_dynamic_offsets, bind_group: bind_group_guard.get_owned(bind_group_id).map_err(|_| { ComputePassError { - scope: PassErrorScope::SetBindGroup(bind_group_id), + scope: PassErrorScope::SetBindGroup, inner: ComputePassErrorInner::InvalidBindGroupId(bind_group_id), } })?, @@ -107,7 +107,7 @@ impl ComputeCommand { pipelines_guard .get_owned(pipeline_id) .map_err(|_| ComputePassError { - scope: PassErrorScope::SetPipelineCompute(pipeline_id), + scope: PassErrorScope::SetPipelineCompute, inner: ComputePassErrorInner::InvalidPipeline(pipeline_id), })?, ), @@ -128,10 +128,7 @@ impl ComputeCommand { ArcComputeCommand::DispatchIndirect { buffer: buffers_guard.get_owned(buffer_id).map_err(|_| { ComputePassError { - scope: PassErrorScope::Dispatch { - indirect: true, - pipeline: None, // TODO: not used right now, but once we do the resolve during recording we can use this again. - }, + scope: PassErrorScope::Dispatch { indirect: true }, inner: ComputePassErrorInner::InvalidBufferId(buffer_id), } })?, diff --git a/wgpu-core/src/command/draw.rs b/wgpu-core/src/command/draw.rs index d9745acc0b..d43039a896 100644 --- a/wgpu-core/src/command/draw.rs +++ b/wgpu-core/src/command/draw.rs @@ -2,7 +2,10 @@ use crate::{ binding_model::{LateMinBufferBindingSizeMismatch, PushConstantUploadError}, error::ErrorFormatter, id, - resource::{DestroyedResourceError, MissingBufferUsageError, MissingTextureUsageError}, + resource::{ + DestroyedResourceError, MissingBufferUsageError, MissingTextureUsageError, + ResourceErrorIdent, + }, track::ResourceUsageCompatibilityError, }; use wgt::VertexStepMode; @@ -10,19 +13,26 @@ use wgt::VertexStepMode; use thiserror::Error; /// Error validating a draw call. -#[derive(Clone, Debug, Error, Eq, PartialEq)] +#[derive(Clone, Debug, Error)] #[non_exhaustive] pub enum DrawError { #[error("Blend constant needs to be set")] MissingBlendConstant, #[error("Render pipeline must be set")] MissingPipeline, - #[error("Vertex buffer {index} must be set")] - MissingVertexBuffer { index: u32 }, + #[error("Currently set {pipeline} requires vertex buffer {index} to be set")] + MissingVertexBuffer { + pipeline: ResourceErrorIdent, + index: u32, + }, #[error("Index buffer must be set")] MissingIndexBuffer, - #[error("Incompatible bind group at index {index} in the current render pipeline")] - IncompatibleBindGroup { index: u32, diff: Vec }, + #[error("Bind group at index {index} is incompatible with the current set {pipeline}")] + IncompatibleBindGroup { + index: u32, + pipeline: ResourceErrorIdent, + diff: Vec, + }, #[error("Vertex {last_vertex} extends beyond limit {vertex_limit} imposed by the buffer in slot {slot}. Did you bind the correct `Vertex` step-rate vertex buffer?")] VertexBeyondLimit { last_vertex: u64, @@ -45,11 +55,12 @@ pub enum DrawError { #[error("Index {last_index} extends beyond limit {index_limit}. Did you bind the correct index buffer?")] IndexBeyondLimit { last_index: u64, index_limit: u64 }, #[error( - "Pipeline index format ({pipeline:?}) and buffer index format ({buffer:?}) do not match" + "Index buffer format {buffer_format:?} doesn't match {pipeline}'s index format {pipeline_format:?}" )] UnmatchedIndexFormats { - pipeline: wgt::IndexFormat, - buffer: wgt::IndexFormat, + pipeline: ResourceErrorIdent, + pipeline_format: wgt::IndexFormat, + buffer_format: wgt::IndexFormat, }, #[error(transparent)] BindingSizeTooSmall(#[from] LateMinBufferBindingSizeMismatch), @@ -72,16 +83,16 @@ pub enum RenderCommandError { VertexBufferIndexOutOfRange { index: u32, max: u32 }, #[error("Dynamic buffer offset {0} does not respect device's requested `{1}` limit {2}")] UnalignedBufferOffset(u64, &'static str, u32), - #[error("Number of buffer offsets ({actual}) does not match the number of dynamic bindings ({expected})")] - InvalidDynamicOffsetCount { actual: usize, expected: usize }, #[error("Render pipeline {0:?} is invalid")] InvalidPipeline(id::RenderPipelineId), #[error("QuerySet {0:?} is invalid")] InvalidQuerySet(id::QuerySetId), #[error("Render pipeline targets are incompatible with render pass")] IncompatiblePipelineTargets(#[from] crate::device::RenderPassCompatibilityError), - #[error("Pipeline writes to depth/stencil, while the pass has read-only depth/stencil")] - IncompatiblePipelineRods, + #[error("{0} writes to depth, while the pass has read-only depth access")] + IncompatibleDepthAccess(ResourceErrorIdent), + #[error("{0} writes to stencil, while the pass has read-only stencil access")] + IncompatibleStencilAccess(ResourceErrorIdent), #[error(transparent)] ResourceUsageCompatibility(#[from] ResourceUsageCompatibilityError), #[error(transparent)] diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index b237a375a6..844691c7c2 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -305,7 +305,6 @@ impl CommandBufferMutable { /// whose contents eventually become the property of the submission queue. pub struct CommandBuffer { pub(crate) device: Arc>, - limits: wgt::Limits, support_clear_texture: bool, pub(crate) info: ResourceInfo>, @@ -344,7 +343,6 @@ impl CommandBuffer { ) -> Self { CommandBuffer { device: device.clone(), - limits: device.limits.clone(), support_clear_texture: device.features.contains(wgt::Features::CLEAR_TEXTURE), info: ResourceInfo::new(label, None), data: Mutex::new( @@ -869,17 +867,17 @@ pub enum PassErrorScope { #[error("In a pass parameter")] Pass(Option), #[error("In a set_bind_group command")] - SetBindGroup(id::BindGroupId), + SetBindGroup, #[error("In a set_pipeline command")] - SetPipelineRender(id::RenderPipelineId), + SetPipelineRender, #[error("In a set_pipeline command")] - SetPipelineCompute(id::ComputePipelineId), + SetPipelineCompute, #[error("In a set_push_constant command")] SetPushConstant, #[error("In a set_vertex_buffer command")] - SetVertexBuffer(id::BufferId), + SetVertexBuffer, #[error("In a set_index_buffer command")] - SetIndexBuffer(id::BufferId), + SetIndexBuffer, #[error("In a set_blend_constant command")] SetBlendConstant, #[error("In a set_stencil_reference command")] @@ -889,11 +887,7 @@ pub enum PassErrorScope { #[error("In a set_scissor_rect command")] SetScissorRect, #[error("In a draw command, kind: {kind:?}")] - Draw { - kind: DrawKind, - indexed: bool, - pipeline: Option, - }, + Draw { kind: DrawKind, indexed: bool }, #[error("While resetting queries after the renderpass was ran")] QueryReset, #[error("In a write_timestamp command")] @@ -909,10 +903,7 @@ pub enum PassErrorScope { #[error("In a execute_bundle command")] ExecuteBundle, #[error("In a dispatch command, indirect:{indirect}")] - Dispatch { - indirect: bool, - pipeline: Option, - }, + Dispatch { indirect: bool }, #[error("In a push_debug_group command")] PushDebugGroup, #[error("In a pop_debug_group command")] @@ -931,31 +922,6 @@ impl PrettyError for PassErrorScope { Self::Pass(Some(id)) => { fmt.command_buffer_label(&id); } - Self::SetBindGroup(id) => { - fmt.bind_group_label(&id); - } - Self::SetPipelineRender(id) => { - fmt.render_pipeline_label(&id); - } - Self::SetPipelineCompute(id) => { - fmt.compute_pipeline_label(&id); - } - Self::SetVertexBuffer(id) => { - fmt.buffer_label(&id); - } - Self::SetIndexBuffer(id) => { - fmt.buffer_label(&id); - } - Self::Draw { - pipeline: Some(id), .. - } => { - fmt.render_pipeline_label(&id); - } - Self::Dispatch { - pipeline: Some(id), .. - } => { - fmt.compute_pipeline_label(&id); - } _ => {} } } diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index 16557a4b4d..4595b70d32 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -10,7 +10,7 @@ use crate::{ id, init_tracker::MemoryInitKind, resource::{DestroyedResourceError, ParentDevice, QuerySet}, - track::TrackerIndex, + track::{StatelessTracker, TrackerIndex}, FastHashMap, }; use std::{iter, marker::PhantomData, sync::Arc}; @@ -124,6 +124,8 @@ impl crate::error::PrettyError for QueryError { #[derive(Clone, Debug, Error)] #[non_exhaustive] pub enum QueryUseError { + #[error(transparent)] + Device(#[from] DeviceError), #[error("Query {query_index} is out of bounds for a query set of size {query_set_size}")] OutOfBounds { query_index: u32, @@ -228,6 +230,7 @@ impl QuerySet { pub(super) fn validate_and_begin_occlusion_query( query_set: Arc>, raw_encoder: &mut A::CommandEncoder, + tracker: &mut StatelessTracker>, query_index: u32, reset_state: Option<&mut QueryResetMap>, active_query: &mut Option<(Arc>, u32)>, @@ -235,6 +238,8 @@ pub(super) fn validate_and_begin_occlusion_query( let needs_reset = reset_state.is_none(); query_set.validate_query(SimplifiedQueryType::Occlusion, query_index, reset_state)?; + tracker.add_single(&query_set); + if let Some((_old, old_idx)) = active_query.take() { return Err(QueryUseError::AlreadyStarted { active_query_index: old_idx, @@ -269,10 +274,14 @@ pub(super) fn end_occlusion_query( pub(super) fn validate_and_begin_pipeline_statistics_query( query_set: Arc>, raw_encoder: &mut A::CommandEncoder, + tracker: &mut StatelessTracker>, + cmd_buf: &CommandBuffer, query_index: u32, reset_state: Option<&mut QueryResetMap>, active_query: &mut Option<(Arc>, u32)>, ) -> Result<(), QueryUseError> { + query_set.same_device_as(cmd_buf)?; + let needs_reset = reset_state.is_none(); query_set.validate_query( SimplifiedQueryType::PipelineStatistics, @@ -280,6 +289,8 @@ pub(super) fn validate_and_begin_pipeline_statistics_query( reset_state, )?; + tracker.add_single(&query_set); + if let Some((_old, old_idx)) = active_query.take() { return Err(QueryUseError::AlreadyStarted { active_query_index: old_idx, diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index ab44dc8798..133bdad26e 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -1,6 +1,9 @@ +use crate::binding_model::BindGroup; use crate::command::{ validate_and_begin_occlusion_query, validate_and_begin_pipeline_statistics_query, }; +use crate::init_tracker::BufferInitTrackerAction; +use crate::pipeline::RenderPipeline; use crate::resource::Resource; use crate::snatch::SnatchGuard; use crate::{ @@ -16,7 +19,7 @@ use crate::{ }, device::{ AttachmentData, Device, DeviceError, MissingDownlevelFlags, MissingFeatures, - RenderPassCompatibilityCheckType, RenderPassCompatibilityError, RenderPassContext, + RenderPassCompatibilityError, RenderPassContext, }, error::{ErrorFormatter, PrettyError}, global::Global, @@ -37,8 +40,8 @@ use arrayvec::ArrayVec; use hal::CommandEncoder as _; use thiserror::Error; use wgt::{ - BufferAddress, BufferSize, BufferUsages, Color, IndexFormat, TextureUsages, - TextureViewDimension, VertexStepMode, + BufferAddress, BufferSize, BufferUsages, Color, DynamicOffset, IndexFormat, ShaderStages, + TextureUsages, TextureViewDimension, VertexStepMode, }; #[cfg(feature = "serde")] @@ -323,32 +326,22 @@ impl OptionalState { #[derive(Debug, Default)] struct IndexState { - bound_buffer_view: Option<(id::BufferId, Range)>, - format: Option, - pipeline_format: Option, + buffer_format: Option, limit: u64, } impl IndexState { - fn update_limit(&mut self) { - self.limit = match self.bound_buffer_view { - Some((_, ref range)) => { - let format = self - .format - .expect("IndexState::update_limit must be called after a index buffer is set"); - let shift = match format { - IndexFormat::Uint16 => 1, - IndexFormat::Uint32 => 2, - }; - - (range.end - range.start) >> shift - } - None => 0, - } + fn update_buffer(&mut self, range: Range, format: IndexFormat) { + self.buffer_format = Some(format); + let shift = match format { + IndexFormat::Uint16 => 1, + IndexFormat::Uint32 => 2, + }; + self.limit = (range.end - range.start) >> shift; } fn reset(&mut self) { - self.bound_buffer_view = None; + self.buffer_format = None; self.limit = 0; } } @@ -383,8 +376,6 @@ struct VertexState { instance_limit: u64, /// Buffer slot which the shortest instance rate vertex buffer is bound to instance_limit_slot: u32, - /// Total amount of buffers required by the pipeline. - buffers_required: u32, } impl VertexState { @@ -440,63 +431,87 @@ impl VertexState { } } -#[derive(Debug)] -struct State { +struct State<'attachment, 'scope, 'snatch_guard, 'cmd_buf, 'raw_encoder, A: HalApi> { pipeline_flags: PipelineFlags, binder: Binder, blend_constant: OptionalState, stencil_reference: u32, - pipeline: Option, + pipeline: Option>>, index: IndexState, vertex: VertexState, debug_scope_depth: u32, + + info: RenderPassInfo<'attachment, 'scope, A>, + + snatch_guard: &'snatch_guard SnatchGuard<'snatch_guard>, + + device: &'cmd_buf Arc>, + + raw_encoder: &'raw_encoder mut A::CommandEncoder, + + tracker: &'cmd_buf mut Tracker, + buffer_memory_init_actions: &'cmd_buf mut Vec>, + texture_memory_actions: &'cmd_buf mut CommandBufferTextureMemoryActions, + + temp_offsets: Vec, + dynamic_offset_count: usize, + string_offset: usize, + active_query: Option<(Arc>, u32)>, } -impl State { +impl<'attachment, 'scope, 'snatch_guard, 'cmd_buf, 'raw_encoder, A: HalApi> + State<'attachment, 'scope, 'snatch_guard, 'cmd_buf, 'raw_encoder, A> +{ fn is_ready(&self, indexed: bool) -> Result<(), DrawError> { - // Determine how many vertex buffers have already been bound - let vertex_buffer_count = self.vertex.inputs.iter().take_while(|v| v.bound).count() as u32; - // Compare with the needed quantity - if vertex_buffer_count < self.vertex.buffers_required { - return Err(DrawError::MissingVertexBuffer { - index: vertex_buffer_count, - }); - } + if let Some(pipeline) = self.pipeline.as_ref() { + let bind_mask = self.binder.invalid_mask(); + if bind_mask != 0 { + return Err(DrawError::IncompatibleBindGroup { + index: bind_mask.trailing_zeros(), + pipeline: pipeline.error_ident(), + diff: self.binder.bgl_diff(), + }); + } + self.binder.check_late_buffer_bindings()?; - let bind_mask = self.binder.invalid_mask(); - if bind_mask != 0 { - //let (expected, provided) = self.binder.entries[index as usize].info(); - return Err(DrawError::IncompatibleBindGroup { - index: bind_mask.trailing_zeros(), - diff: self.binder.bgl_diff(), - }); - } - if self.pipeline.is_none() { - return Err(DrawError::MissingPipeline); - } - if self.blend_constant == OptionalState::Required { - return Err(DrawError::MissingBlendConstant); - } + if self.blend_constant == OptionalState::Required { + return Err(DrawError::MissingBlendConstant); + } - if indexed { - // Pipeline expects an index buffer - if let Some(pipeline_index_format) = self.index.pipeline_format { - // We have a buffer bound - let buffer_index_format = self.index.format.ok_or(DrawError::MissingIndexBuffer)?; - - // The buffers are different formats - if pipeline_index_format != buffer_index_format { - return Err(DrawError::UnmatchedIndexFormats { - pipeline: pipeline_index_format, - buffer: buffer_index_format, - }); + // Determine how many vertex buffers have already been bound + let vertex_buffer_count = + self.vertex.inputs.iter().take_while(|v| v.bound).count() as u32; + // Compare with the needed quantity + if vertex_buffer_count < pipeline.vertex_steps.len() as u32 { + return Err(DrawError::MissingVertexBuffer { + pipeline: pipeline.error_ident(), + index: vertex_buffer_count, + }); + } + + if indexed { + // Pipeline expects an index buffer + if let Some(pipeline_index_format) = pipeline.strip_index_format { + // We have a buffer bound + let buffer_index_format = self + .index + .buffer_format + .ok_or(DrawError::MissingIndexBuffer)?; + + // The buffers are different formats + if pipeline_index_format != buffer_index_format { + return Err(DrawError::UnmatchedIndexFormats { + pipeline: pipeline.error_ident(), + pipeline_format: pipeline_index_format, + buffer_format: buffer_index_format, + }); + } } } + Ok(()) + } else { + Err(DrawError::MissingPipeline) } - - self.binder.check_late_buffer_bindings()?; - - Ok(()) } /// Reset the `RenderBundle`-related states. @@ -815,7 +830,7 @@ impl<'a, 'd, A: HalApi> RenderPassInfo<'a, 'd, A> { color_attachments: &[Option], depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, timestamp_writes: Option<&RenderPassTimestampWrites>, - occlusion_query_set: Option, + occlusion_query_set: Option>>, encoder: &mut CommandEncoder, trackers: &mut Tracker, texture_memory_actions: &mut CommandBufferTextureMemoryActions, @@ -1210,13 +1225,8 @@ impl<'a, 'd, A: HalApi> RenderPassInfo<'a, 'd, A> { None }; - let occlusion_query_set = if let Some(occlusion_query_set) = occlusion_query_set { - let query_set = query_set_guard - .get(occlusion_query_set) - .map_err(|_| RenderPassErrorInner::InvalidQuerySet(occlusion_query_set))?; - - trackers.query_sets.add_single(query_set); - + let occlusion_query_set = if let Some(query_set) = occlusion_query_set { + let query_set = trackers.query_sets.insert_single(query_set); Some(query_set.raw.as_ref().unwrap()) } else { None @@ -1352,7 +1362,20 @@ impl Global { timestamp_writes: Option<&RenderPassTimestampWrites>, occlusion_query_set_id: Option, ) -> Result<(), RenderPassError> { - let commands = RenderCommand::resolve_render_command_ids(A::hub(self), &base.commands)?; + let pass_scope = PassErrorScope::PassEncoder(encoder_id); + + let hub = A::hub(self); + + let commands = RenderCommand::resolve_render_command_ids(hub, &base.commands)?; + + let occlusion_query_set = occlusion_query_set_id + .map(|id| { + hub.query_sets + .get(id) + .map_err(|_| RenderPassErrorInner::InvalidQuerySet(id)) + }) + .transpose() + .map_pass_err(pass_scope)?; self.render_pass_end_impl::( encoder_id, @@ -1366,7 +1389,7 @@ impl Global { color_attachments, depth_stencil_attachment, timestamp_writes, - occlusion_query_set_id, + occlusion_query_set, ) } @@ -1378,17 +1401,13 @@ impl Global { color_attachments: &[Option], depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, timestamp_writes: Option<&RenderPassTimestampWrites>, - occlusion_query_set_id: Option, + occlusion_query_set: Option>>, ) -> Result<(), RenderPassError> { profiling::scope!( "CommandEncoder::run_render_pass {}", base.label.unwrap_or("") ); - let discard_hal_labels = self - .instance - .flags - .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS); let hal_label = hal_label(base.label.as_deref(), self.instance.flags); let pass_scope = PassErrorScope::PassEncoder(encoder_id); @@ -1398,7 +1417,7 @@ impl Global { let cmd_buf: Arc> = CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?; let device = &cmd_buf.device; - let snatch_guard = device.snatchable_lock.read(); + let snatch_guard = &device.snatchable_lock.read(); let (scope, pending_discard_init_fixups) = { let mut cmd_buf_data = cmd_buf.data.lock(); @@ -1417,7 +1436,9 @@ impl Global { target_colors: color_attachments.to_vec(), target_depth_stencil: depth_stencil_attachment.cloned(), timestamp_writes: timestamp_writes.cloned(), - occlusion_query_set_id, + occlusion_query_set_id: occlusion_query_set + .as_ref() + .map(|query_set| query_set.as_info().id()), }); } @@ -1446,20 +1467,20 @@ impl Global { encoder_id ); - let mut info = RenderPassInfo::start( + let info = RenderPassInfo::start( device, hal_label, color_attachments, depth_stencil_attachment, timestamp_writes, - occlusion_query_set_id, + occlusion_query_set.clone(), encoder, tracker, texture_memory_actions, pending_query_resets, &*view_guard, &*query_set_guard, - &snatch_guard, + snatch_guard, ) .map_pass_err(pass_scope)?; @@ -1485,11 +1506,22 @@ impl Global { index: IndexState::default(), vertex: VertexState::default(), debug_scope_depth: 0, + + info, + + snatch_guard, + + device, + raw_encoder: raw, + tracker, + buffer_memory_init_actions, + texture_memory_actions, + + temp_offsets: Vec::new(), + dynamic_offset_count: 0, + string_offset: 0, + active_query: None, }; - let mut temp_offsets = Vec::new(); - let mut dynamic_offset_count = 0; - let mut string_offset = 0; - let mut active_query = None; for command in base.commands { match command { @@ -1498,202 +1530,20 @@ impl Global { num_dynamic_offsets, bind_group, } => { - let bind_group_id = bind_group.as_info().id(); - api_log!("RenderPass::set_bind_group {index} {bind_group_id:?}"); - - let scope = PassErrorScope::SetBindGroup(bind_group_id); - let max_bind_groups = device.limits.max_bind_groups; - if index >= max_bind_groups { - return Err(RenderCommandError::BindGroupIndexOutOfRange { - index, - max: max_bind_groups, - }) - .map_pass_err(scope); - } - - temp_offsets.clear(); - temp_offsets.extend_from_slice( - &base.dynamic_offsets - [dynamic_offset_count..dynamic_offset_count + num_dynamic_offsets], - ); - dynamic_offset_count += num_dynamic_offsets; - - let bind_group = tracker.bind_groups.insert_single(bind_group); - - bind_group - .same_device_as(cmd_buf.as_ref()) - .map_pass_err(scope)?; - - bind_group - .validate_dynamic_bindings(index, &temp_offsets, &cmd_buf.limits) - .map_pass_err(scope)?; - - // merge the resource tracker in - unsafe { - info.usage_scope - .merge_bind_group(&bind_group.used) - .map_pass_err(scope)?; - } - //Note: stateless trackers are not merged: the lifetime reference - // is held to the bind group itself. - - buffer_memory_init_actions.extend( - bind_group.used_buffer_ranges.iter().filter_map(|action| { - action - .buffer - .initialization_status - .read() - .check_action(action) - }), - ); - for action in bind_group.used_texture_ranges.iter() { - info.pending_discard_init_fixups - .extend(texture_memory_actions.register_init_action(action)); - } - - let pipeline_layout = state.binder.pipeline_layout.clone(); - let entries = - state - .binder - .assign_group(index as usize, bind_group, &temp_offsets); - if !entries.is_empty() && pipeline_layout.is_some() { - let pipeline_layout = pipeline_layout.as_ref().unwrap().raw(); - for (i, e) in entries.iter().enumerate() { - if let Some(group) = e.group.as_ref() { - let raw_bg = - group.try_raw(&snatch_guard).map_pass_err(scope)?; - unsafe { - raw.set_bind_group( - pipeline_layout, - index + i as u32, - raw_bg, - &e.dynamic_offsets, - ); - } - } - } - } + let scope = PassErrorScope::SetBindGroup; + set_bind_group( + &mut state, + &cmd_buf, + &base.dynamic_offsets, + index, + num_dynamic_offsets, + bind_group, + ) + .map_pass_err(scope)?; } ArcRenderCommand::SetPipeline(pipeline) => { - let pipeline_id = pipeline.as_info().id(); - api_log!("RenderPass::set_pipeline {pipeline_id:?}"); - - let scope = PassErrorScope::SetPipelineRender(pipeline_id); - state.pipeline = Some(pipeline_id); - - let pipeline = tracker.render_pipelines.insert_single(pipeline); - - pipeline - .same_device_as(cmd_buf.as_ref()) - .map_pass_err(scope)?; - - info.context - .check_compatible( - &pipeline.pass_context, - RenderPassCompatibilityCheckType::RenderPipeline, - ) - .map_err(RenderCommandError::IncompatiblePipelineTargets) - .map_pass_err(scope)?; - - state.pipeline_flags = pipeline.flags; - - if (pipeline.flags.contains(PipelineFlags::WRITES_DEPTH) - && info.is_depth_read_only) - || (pipeline.flags.contains(PipelineFlags::WRITES_STENCIL) - && info.is_stencil_read_only) - { - return Err(RenderCommandError::IncompatiblePipelineRods) - .map_pass_err(scope); - } - - state - .blend_constant - .require(pipeline.flags.contains(PipelineFlags::BLEND_CONSTANT)); - - unsafe { - raw.set_render_pipeline(pipeline.raw()); - } - - if pipeline.flags.contains(PipelineFlags::STENCIL_REFERENCE) { - unsafe { - raw.set_stencil_reference(state.stencil_reference); - } - } - - // Rebind resource - if state.binder.pipeline_layout.is_none() - || !state - .binder - .pipeline_layout - .as_ref() - .unwrap() - .is_equal(&pipeline.layout) - { - let (start_index, entries) = state.binder.change_pipeline_layout( - &pipeline.layout, - &pipeline.late_sized_buffer_groups, - ); - if !entries.is_empty() { - for (i, e) in entries.iter().enumerate() { - if let Some(group) = e.group.as_ref() { - let raw_bg = - group.try_raw(&snatch_guard).map_pass_err(scope)?; - unsafe { - raw.set_bind_group( - pipeline.layout.raw(), - start_index as u32 + i as u32, - raw_bg, - &e.dynamic_offsets, - ); - } - } - } - } - - // Clear push constant ranges - let non_overlapping = super::bind::compute_nonoverlapping_ranges( - &pipeline.layout.push_constant_ranges, - ); - for range in non_overlapping { - let offset = range.range.start; - let size_bytes = range.range.end - offset; - super::push_constant_clear( - offset, - size_bytes, - |clear_offset, clear_data| unsafe { - raw.set_push_constants( - pipeline.layout.raw(), - range.stages, - clear_offset, - clear_data, - ); - }, - ); - } - } - - state.index.pipeline_format = pipeline.strip_index_format; - - let vertex_steps_len = pipeline.vertex_steps.len(); - state.vertex.buffers_required = vertex_steps_len as u32; - - // Initialize each `vertex.inputs[i].step` from - // `pipeline.vertex_steps[i]`. Enlarge `vertex.inputs` - // as necessary to accommodate all slots in the - // pipeline. If `vertex.inputs` is longer, fill the - // extra entries with default `VertexStep`s. - while state.vertex.inputs.len() < vertex_steps_len { - state.vertex.inputs.push(VertexBufferState::EMPTY); - } - - // This is worse as a `zip`, but it's close. - let mut steps = pipeline.vertex_steps.iter(); - for input in state.vertex.inputs.iter_mut() { - input.step = steps.next().cloned().unwrap_or_default(); - } - - // Update vertex buffer limits. - state.vertex.update_limits(); + let scope = PassErrorScope::SetPipelineRender; + set_pipeline(&mut state, &cmd_buf, pipeline).map_pass_err(scope)?; } ArcRenderCommand::SetIndexBuffer { buffer, @@ -1701,50 +1551,9 @@ impl Global { offset, size, } => { - let buffer_id = buffer.as_info().id(); - api_log!("RenderPass::set_index_buffer {buffer_id:?}"); - - let scope = PassErrorScope::SetIndexBuffer(buffer_id); - - info.usage_scope - .buffers - .merge_single(&buffer, hal::BufferUses::INDEX) - .map_pass_err(scope)?; - - buffer - .same_device_as(cmd_buf.as_ref()) + let scope = PassErrorScope::SetIndexBuffer; + set_index_buffer(&mut state, &cmd_buf, buffer, index_format, offset, size) .map_pass_err(scope)?; - - buffer - .check_usage(BufferUsages::INDEX) - .map_pass_err(scope)?; - let buf_raw = buffer.try_raw(&snatch_guard).map_pass_err(scope)?; - - let end = match size { - Some(s) => offset + s.get(), - None => buffer.size, - }; - state.index.bound_buffer_view = Some((buffer_id, offset..end)); - - state.index.format = Some(index_format); - state.index.update_limit(); - - buffer_memory_init_actions.extend( - buffer.initialization_status.read().create_action( - &buffer, - offset..end, - MemoryInitKind::NeedsInitializedMemory, - ), - ); - - let bb = hal::BufferBinding { - buffer: buf_raw, - offset, - size, - }; - unsafe { - raw.set_index_buffer(bb, index_format); - } } ArcRenderCommand::SetVertexBuffer { slot, @@ -1752,129 +1561,23 @@ impl Global { offset, size, } => { - let buffer_id = buffer.as_info().id(); - api_log!("RenderPass::set_vertex_buffer {slot} {buffer_id:?}"); - - let scope = PassErrorScope::SetVertexBuffer(buffer_id); - - info.usage_scope - .buffers - .merge_single(&buffer, hal::BufferUses::VERTEX) - .map_pass_err(scope)?; - - buffer - .same_device_as(cmd_buf.as_ref()) - .map_pass_err(scope)?; - - let max_vertex_buffers = device.limits.max_vertex_buffers; - if slot >= max_vertex_buffers { - return Err(RenderCommandError::VertexBufferIndexOutOfRange { - index: slot, - max: max_vertex_buffers, - }) - .map_pass_err(scope); - } - - buffer - .check_usage(BufferUsages::VERTEX) + let scope = PassErrorScope::SetVertexBuffer; + set_vertex_buffer(&mut state, &cmd_buf, slot, buffer, offset, size) .map_pass_err(scope)?; - let buf_raw = buffer.try_raw(&snatch_guard).map_pass_err(scope)?; - - let empty_slots = - (1 + slot as usize).saturating_sub(state.vertex.inputs.len()); - state - .vertex - .inputs - .extend(iter::repeat(VertexBufferState::EMPTY).take(empty_slots)); - let vertex_state = &mut state.vertex.inputs[slot as usize]; - //TODO: where are we checking that the offset is in bound? - vertex_state.total_size = match size { - Some(s) => s.get(), - None => buffer.size - offset, - }; - vertex_state.bound = true; - - buffer_memory_init_actions.extend( - buffer.initialization_status.read().create_action( - &buffer, - offset..(offset + vertex_state.total_size), - MemoryInitKind::NeedsInitializedMemory, - ), - ); - - let bb = hal::BufferBinding { - buffer: buf_raw, - offset, - size, - }; - unsafe { - raw.set_vertex_buffer(slot, bb); - } - state.vertex.update_limits(); } ArcRenderCommand::SetBlendConstant(ref color) => { - api_log!("RenderPass::set_blend_constant"); - - state.blend_constant = OptionalState::Set; - let array = [ - color.r as f32, - color.g as f32, - color.b as f32, - color.a as f32, - ]; - unsafe { - raw.set_blend_constants(&array); - } + set_blend_constant(&mut state, color); } ArcRenderCommand::SetStencilReference(value) => { - api_log!("RenderPass::set_stencil_reference {value}"); - - state.stencil_reference = value; - if state - .pipeline_flags - .contains(PipelineFlags::STENCIL_REFERENCE) - { - unsafe { - raw.set_stencil_reference(value); - } - } + set_stencil_reference(&mut state, value); } ArcRenderCommand::SetViewport { - ref rect, + rect, depth_min, depth_max, } => { - api_log!("RenderPass::set_viewport {rect:?}"); - let scope = PassErrorScope::SetViewport; - if rect.x < 0.0 - || rect.y < 0.0 - || rect.w <= 0.0 - || rect.h <= 0.0 - || rect.x + rect.w > info.extent.width as f32 - || rect.y + rect.h > info.extent.height as f32 - { - return Err(RenderCommandError::InvalidViewportRect( - *rect, - info.extent, - )) - .map_pass_err(scope); - } - if !(0.0..=1.0).contains(&depth_min) || !(0.0..=1.0).contains(&depth_max) { - return Err(RenderCommandError::InvalidViewportDepth( - depth_min, depth_max, - )) - .map_pass_err(scope); - } - let r = hal::Rect { - x: rect.x, - y: rect.y, - w: rect.w, - h: rect.h, - }; - unsafe { - raw.set_viewport(&r, depth_min..depth_max); - } + set_viewport(&mut state, rect, depth_min, depth_max).map_pass_err(scope)?; } ArcRenderCommand::SetPushConstant { stages, @@ -1882,59 +1585,20 @@ impl Global { size_bytes, values_offset, } => { - api_log!("RenderPass::set_push_constants"); - let scope = PassErrorScope::SetPushConstant; - let values_offset = values_offset - .ok_or(RenderPassErrorInner::InvalidValuesOffset) - .map_pass_err(scope)?; - - let end_offset_bytes = offset + size_bytes; - let values_end_offset = - (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; - let data_slice = - &base.push_constant_data[(values_offset as usize)..values_end_offset]; - - let pipeline_layout = state - .binder - .pipeline_layout - .as_ref() - .ok_or(DrawError::MissingPipeline) - .map_pass_err(scope)?; - - pipeline_layout - .validate_push_constant_ranges(stages, offset, end_offset_bytes) - .map_err(RenderCommandError::from) - .map_pass_err(scope)?; - - unsafe { - raw.set_push_constants( - pipeline_layout.raw(), - stages, - offset, - data_slice, - ) - } + set_push_constant( + &mut state, + &base.push_constant_data, + stages, + offset, + size_bytes, + values_offset, + ) + .map_pass_err(scope)?; } - ArcRenderCommand::SetScissor(ref rect) => { - api_log!("RenderPass::set_scissor_rect {rect:?}"); - + ArcRenderCommand::SetScissor(rect) => { let scope = PassErrorScope::SetScissorRect; - if rect.x + rect.w > info.extent.width - || rect.y + rect.h > info.extent.height - { - return Err(RenderCommandError::InvalidScissorRect(*rect, info.extent)) - .map_pass_err(scope); - } - let r = hal::Rect { - x: rect.x, - y: rect.y, - w: rect.w, - h: rect.h, - }; - unsafe { - raw.set_scissor_rect(&r); - } + set_scissor(&mut state, rect).map_pass_err(scope)?; } ArcRenderCommand::Draw { vertex_count, @@ -1942,49 +1606,18 @@ impl Global { first_vertex, first_instance, } => { - api_log!( - "RenderPass::draw {vertex_count} {instance_count} {first_vertex} {first_instance}" - ); - - let indexed = false; let scope = PassErrorScope::Draw { kind: DrawKind::Draw, - indexed, - pipeline: state.pipeline, + indexed: false, }; - state.is_ready(indexed).map_pass_err(scope)?; - - let last_vertex = first_vertex as u64 + vertex_count as u64; - let vertex_limit = state.vertex.vertex_limit; - if last_vertex > vertex_limit { - return Err(DrawError::VertexBeyondLimit { - last_vertex, - vertex_limit, - slot: state.vertex.vertex_limit_slot, - }) - .map_pass_err(scope); - } - let last_instance = first_instance as u64 + instance_count as u64; - let instance_limit = state.vertex.instance_limit; - if last_instance > instance_limit { - return Err(DrawError::InstanceBeyondLimit { - last_instance, - instance_limit, - slot: state.vertex.instance_limit_slot, - }) - .map_pass_err(scope); - } - - unsafe { - if instance_count > 0 && vertex_count > 0 { - raw.draw( - first_vertex, - vertex_count, - first_instance, - instance_count, - ); - } - } + draw( + &mut state, + vertex_count, + instance_count, + first_vertex, + first_instance, + ) + .map_pass_err(scope)?; } ArcRenderCommand::DrawIndexed { index_count, @@ -1993,57 +1626,26 @@ impl Global { base_vertex, first_instance, } => { - api_log!("RenderPass::draw_indexed {index_count} {instance_count} {first_index} {base_vertex} {first_instance}"); - - let indexed = true; let scope = PassErrorScope::Draw { kind: DrawKind::Draw, - indexed, - pipeline: state.pipeline, + indexed: true, }; - state.is_ready(indexed).map_pass_err(scope)?; - - let last_index = first_index as u64 + index_count as u64; - let index_limit = state.index.limit; - if last_index > index_limit { - return Err(DrawError::IndexBeyondLimit { - last_index, - index_limit, - }) - .map_pass_err(scope); - } - let last_instance = first_instance as u64 + instance_count as u64; - let instance_limit = state.vertex.instance_limit; - if last_instance > instance_limit { - return Err(DrawError::InstanceBeyondLimit { - last_instance, - instance_limit, - slot: state.vertex.instance_limit_slot, - }) - .map_pass_err(scope); - } - - unsafe { - if instance_count > 0 && index_count > 0 { - raw.draw_indexed( - first_index, - index_count, - base_vertex, - first_instance, - instance_count, - ); - } - } + draw_indexed( + &mut state, + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + ) + .map_pass_err(scope)?; } ArcRenderCommand::MultiDrawIndirect { - buffer: indirect_buffer, + buffer, offset, count, indexed, } => { - let indirect_buffer_id = indirect_buffer.as_info().id(); - api_log!("RenderPass::draw_indirect (indexed:{indexed}) {indirect_buffer_id:?} {offset} {count:?}"); - let scope = PassErrorScope::Draw { kind: if count.is_some() { DrawKind::MultiDrawIndirect @@ -2051,260 +1653,72 @@ impl Global { DrawKind::DrawIndirect }, indexed, - pipeline: state.pipeline, - }; - state.is_ready(indexed).map_pass_err(scope)?; - - let stride = match indexed { - false => mem::size_of::(), - true => mem::size_of::(), }; - - if count.is_some() { - device - .require_features(wgt::Features::MULTI_DRAW_INDIRECT) - .map_pass_err(scope)?; - } - device - .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) - .map_pass_err(scope)?; - - info.usage_scope - .buffers - .merge_single(&indirect_buffer, hal::BufferUses::INDIRECT) - .map_pass_err(scope)?; - - indirect_buffer - .check_usage(BufferUsages::INDIRECT) + multi_draw_indirect(&mut state, buffer, offset, count, indexed) .map_pass_err(scope)?; - let indirect_raw = - indirect_buffer.try_raw(&snatch_guard).map_pass_err(scope)?; - - let actual_count = count.map_or(1, |c| c.get()); - - let end_offset = offset + stride as u64 * actual_count as u64; - if end_offset > indirect_buffer.size { - return Err(RenderPassErrorInner::IndirectBufferOverrun { - count, - offset, - end_offset, - buffer_size: indirect_buffer.size, - }) - .map_pass_err(scope); - } - - buffer_memory_init_actions.extend( - indirect_buffer.initialization_status.read().create_action( - &indirect_buffer, - offset..end_offset, - MemoryInitKind::NeedsInitializedMemory, - ), - ); - - match indexed { - false => unsafe { - raw.draw_indirect(indirect_raw, offset, actual_count); - }, - true => unsafe { - raw.draw_indexed_indirect(indirect_raw, offset, actual_count); - }, - } } ArcRenderCommand::MultiDrawIndirectCount { - buffer: indirect_buffer, + buffer, offset, count_buffer, count_buffer_offset, max_count, indexed, } => { - let indirect_buffer_id = indirect_buffer.as_info().id(); - let count_buffer_id = count_buffer.as_info().id(); - api_log!("RenderPass::multi_draw_indirect_count (indexed:{indexed}) {indirect_buffer_id:?} {offset} {count_buffer_id:?} {count_buffer_offset:?} {max_count:?}"); - let scope = PassErrorScope::Draw { kind: DrawKind::MultiDrawIndirectCount, indexed, - pipeline: state.pipeline, }; - state.is_ready(indexed).map_pass_err(scope)?; - - let stride = match indexed { - false => mem::size_of::(), - true => mem::size_of::(), - } as u64; - - device - .require_features(wgt::Features::MULTI_DRAW_INDIRECT_COUNT) - .map_pass_err(scope)?; - device - .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) - .map_pass_err(scope)?; - - info.usage_scope - .buffers - .merge_single(&indirect_buffer, hal::BufferUses::INDIRECT) - .map_pass_err(scope)?; - - indirect_buffer - .check_usage(BufferUsages::INDIRECT) - .map_pass_err(scope)?; - let indirect_raw = - indirect_buffer.try_raw(&snatch_guard).map_pass_err(scope)?; - - info.usage_scope - .buffers - .merge_single(&count_buffer, hal::BufferUses::INDIRECT) - .map_pass_err(scope)?; - - count_buffer - .check_usage(BufferUsages::INDIRECT) - .map_pass_err(scope)?; - let count_raw = count_buffer.try_raw(&snatch_guard).map_pass_err(scope)?; - - let end_offset = offset + stride * max_count as u64; - if end_offset > indirect_buffer.size { - return Err(RenderPassErrorInner::IndirectBufferOverrun { - count: None, - offset, - end_offset, - buffer_size: indirect_buffer.size, - }) - .map_pass_err(scope); - } - buffer_memory_init_actions.extend( - indirect_buffer.initialization_status.read().create_action( - &indirect_buffer, - offset..end_offset, - MemoryInitKind::NeedsInitializedMemory, - ), - ); - - let begin_count_offset = count_buffer_offset; - let end_count_offset = count_buffer_offset + 4; - if end_count_offset > count_buffer.size { - return Err(RenderPassErrorInner::IndirectCountBufferOverrun { - begin_count_offset, - end_count_offset, - count_buffer_size: count_buffer.size, - }) - .map_pass_err(scope); - } - buffer_memory_init_actions.extend( - count_buffer.initialization_status.read().create_action( - &count_buffer, - count_buffer_offset..end_count_offset, - MemoryInitKind::NeedsInitializedMemory, - ), - ); - - match indexed { - false => unsafe { - raw.draw_indirect_count( - indirect_raw, - offset, - count_raw, - count_buffer_offset, - max_count, - ); - }, - true => unsafe { - raw.draw_indexed_indirect_count( - indirect_raw, - offset, - count_raw, - count_buffer_offset, - max_count, - ); - }, - } + multi_draw_indirect_count( + &mut state, + buffer, + offset, + count_buffer, + count_buffer_offset, + max_count, + indexed, + ) + .map_pass_err(scope)?; } ArcRenderCommand::PushDebugGroup { color: _, len } => { - state.debug_scope_depth += 1; - if !discard_hal_labels { - let label = str::from_utf8( - &base.string_data[string_offset..string_offset + len], - ) - .unwrap(); - - api_log!("RenderPass::push_debug_group {label:?}"); - unsafe { - raw.begin_debug_marker(label); - } - } - string_offset += len; + push_debug_group(&mut state, &base.string_data, len); } ArcRenderCommand::PopDebugGroup => { - api_log!("RenderPass::pop_debug_group"); - let scope = PassErrorScope::PopDebugGroup; - if state.debug_scope_depth == 0 { - return Err(RenderPassErrorInner::InvalidPopDebugGroup) - .map_pass_err(scope); - } - state.debug_scope_depth -= 1; - if !discard_hal_labels { - unsafe { - raw.end_debug_marker(); - } - } + pop_debug_group(&mut state).map_pass_err(scope)?; } ArcRenderCommand::InsertDebugMarker { color: _, len } => { - if !discard_hal_labels { - let label = str::from_utf8( - &base.string_data[string_offset..string_offset + len], - ) - .unwrap(); - api_log!("RenderPass::insert_debug_marker {label:?}"); - unsafe { - raw.insert_debug_marker(label); - } - } - string_offset += len; + insert_debug_marker(&mut state, &base.string_data, len); } ArcRenderCommand::WriteTimestamp { query_set, query_index, } => { - let query_set_id = query_set.as_info().id(); - api_log!("RenderPass::write_timestamps {query_set_id:?} {query_index}"); let scope = PassErrorScope::WriteTimestamp; - - device - .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES) - .map_pass_err(scope)?; - - let query_set = tracker.query_sets.insert_single(query_set); - - query_set - .validate_and_write_timestamp( - raw, - query_index, - Some(&mut cmd_buf_data.pending_query_resets), - ) - .map_pass_err(scope)?; + write_timestamp( + &mut state, + &mut cmd_buf_data.pending_query_resets, + query_set, + query_index, + ) + .map_pass_err(scope)?; } ArcRenderCommand::BeginOcclusionQuery { query_index } => { api_log!("RenderPass::begin_occlusion_query {query_index}"); let scope = PassErrorScope::BeginOcclusionQuery; - let query_set_id = occlusion_query_set_id + let query_set = occlusion_query_set + .clone() .ok_or(RenderPassErrorInner::MissingOcclusionQuerySet) .map_pass_err(scope)?; - let query_set = query_set_guard - .get(query_set_id) - .map_err(|_| RenderPassErrorInner::InvalidQuerySet(query_set_id)) - .map_pass_err(scope)?; - - tracker.query_sets.add_single(query_set); - validate_and_begin_occlusion_query( - query_set.clone(), - raw, + query_set, + state.raw_encoder, + &mut state.tracker.query_sets, query_index, Some(&mut cmd_buf_data.pending_query_resets), - &mut active_query, + &mut state.active_query, ) .map_pass_err(scope)?; } @@ -2312,24 +1726,27 @@ impl Global { api_log!("RenderPass::end_occlusion_query"); let scope = PassErrorScope::EndOcclusionQuery; - end_occlusion_query(raw, &mut active_query).map_pass_err(scope)?; + end_occlusion_query(state.raw_encoder, &mut state.active_query) + .map_pass_err(scope)?; } ArcRenderCommand::BeginPipelineStatisticsQuery { query_set, query_index, } => { - let query_set_id = query_set.as_info().id(); - api_log!("RenderPass::begin_pipeline_statistics_query {query_set_id:?} {query_index}"); + api_log!( + "RenderPass::begin_pipeline_statistics_query {query_index} {}", + query_set.error_ident() + ); let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set = tracker.query_sets.insert_single(query_set); - validate_and_begin_pipeline_statistics_query( - query_set.clone(), - raw, + query_set, + state.raw_encoder, + &mut state.tracker.query_sets, + cmd_buf.as_ref(), query_index, Some(&mut cmd_buf_data.pending_query_resets), - &mut active_query, + &mut state.active_query, ) .map_pass_err(scope)?; } @@ -2337,88 +1754,21 @@ impl Global { api_log!("RenderPass::end_pipeline_statistics_query"); let scope = PassErrorScope::EndPipelineStatisticsQuery; - end_pipeline_statistics_query(raw, &mut active_query) + end_pipeline_statistics_query(state.raw_encoder, &mut state.active_query) .map_pass_err(scope)?; } ArcRenderCommand::ExecuteBundle(bundle) => { - let bundle_id = bundle.as_info().id(); - api_log!("RenderPass::execute_bundle {bundle_id:?}"); let scope = PassErrorScope::ExecuteBundle; - - // Have to clone the bundle arc, otherwise we keep a mutable reference to the bundle - // while later trying to add the bundle's resources to the tracker. - let bundle = tracker.bundles.insert_single(bundle).clone(); - - bundle - .same_device_as(cmd_buf.as_ref()) - .map_pass_err(scope)?; - - info.context - .check_compatible( - &bundle.context, - RenderPassCompatibilityCheckType::RenderBundle, - ) - .map_err(RenderPassErrorInner::IncompatibleBundleTargets) - .map_pass_err(scope)?; - - if (info.is_depth_read_only && !bundle.is_depth_read_only) - || (info.is_stencil_read_only && !bundle.is_stencil_read_only) - { - return Err( - RenderPassErrorInner::IncompatibleBundleReadOnlyDepthStencil { - pass_depth: info.is_depth_read_only, - pass_stencil: info.is_stencil_read_only, - bundle_depth: bundle.is_depth_read_only, - bundle_stencil: bundle.is_stencil_read_only, - }, - ) - .map_pass_err(scope); - } - - buffer_memory_init_actions.extend( - bundle - .buffer_memory_init_actions - .iter() - .filter_map(|action| { - action - .buffer - .initialization_status - .read() - .check_action(action) - }), - ); - for action in bundle.texture_memory_init_actions.iter() { - info.pending_discard_init_fixups - .extend(texture_memory_actions.register_init_action(action)); - } - - unsafe { bundle.execute(raw, &snatch_guard) } - .map_err(|e| match e { - ExecutionError::DestroyedResource(e) => { - RenderCommandError::DestroyedResource(e) - } - ExecutionError::Unimplemented(what) => { - RenderCommandError::Unimplemented(what) - } - }) - .map_pass_err(scope)?; - - unsafe { - info.usage_scope - .merge_render_bundle(&bundle.used) - .map_pass_err(scope)?; - tracker - .add_from_render_bundle(&bundle.used) - .map_pass_err(scope)?; - }; - state.reset_bundle(); + execute_bundle(&mut state, &cmd_buf, bundle).map_pass_err(scope)?; } } } log::trace!("Merging renderpass into cmd_buf {:?}", encoder_id); - let (trackers, pending_discard_init_fixups) = - info.finish(raw, &snatch_guard).map_pass_err(pass_scope)?; + let (trackers, pending_discard_init_fixups) = state + .info + .finish(state.raw_encoder, state.snatch_guard) + .map_pass_err(pass_scope)?; encoder.close().map_pass_err(pass_scope)?; (trackers, pending_discard_init_fixups) @@ -2443,12 +1793,12 @@ impl Global { transit, &mut tracker.textures, &cmd_buf.device, - &snatch_guard, + snatch_guard, ); cmd_buf_data.pending_query_resets.reset_queries(transit); - CommandBuffer::insert_barriers_from_scope(transit, tracker, &scope, &snatch_guard); + CommandBuffer::insert_barriers_from_scope(transit, tracker, &scope, snatch_guard); } *status = CommandEncoderStatus::Recording; @@ -2458,15 +1808,836 @@ impl Global { } } +fn set_bind_group( + state: &mut State, + cmd_buf: &Arc>, + dynamic_offsets: &[DynamicOffset], + index: u32, + num_dynamic_offsets: usize, + bind_group: Arc>, +) -> Result<(), RenderPassErrorInner> { + api_log!( + "RenderPass::set_bind_group {index} {}", + bind_group.error_ident() + ); + + let max_bind_groups = state.device.limits.max_bind_groups; + if index >= max_bind_groups { + return Err(RenderCommandError::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + } + .into()); + } + + state.temp_offsets.clear(); + state.temp_offsets.extend_from_slice( + &dynamic_offsets + [state.dynamic_offset_count..state.dynamic_offset_count + num_dynamic_offsets], + ); + state.dynamic_offset_count += num_dynamic_offsets; + + let bind_group = state.tracker.bind_groups.insert_single(bind_group); + + bind_group.same_device_as(cmd_buf.as_ref())?; + + bind_group.validate_dynamic_bindings(index, &state.temp_offsets)?; + + // merge the resource tracker in + unsafe { + state.info.usage_scope.merge_bind_group(&bind_group.used)?; + } + //Note: stateless trackers are not merged: the lifetime reference + // is held to the bind group itself. + + state + .buffer_memory_init_actions + .extend(bind_group.used_buffer_ranges.iter().filter_map(|action| { + action + .buffer + .initialization_status + .read() + .check_action(action) + })); + for action in bind_group.used_texture_ranges.iter() { + state + .info + .pending_discard_init_fixups + .extend(state.texture_memory_actions.register_init_action(action)); + } + + let pipeline_layout = state.binder.pipeline_layout.clone(); + let entries = state + .binder + .assign_group(index as usize, bind_group, &state.temp_offsets); + if !entries.is_empty() && pipeline_layout.is_some() { + let pipeline_layout = pipeline_layout.as_ref().unwrap().raw(); + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group.try_raw(state.snatch_guard)?; + unsafe { + state.raw_encoder.set_bind_group( + pipeline_layout, + index + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + Ok(()) +} + +fn set_pipeline( + state: &mut State, + cmd_buf: &Arc>, + pipeline: Arc>, +) -> Result<(), RenderPassErrorInner> { + api_log!("RenderPass::set_pipeline {}", pipeline.error_ident()); + + state.pipeline = Some(pipeline.clone()); + + let pipeline = state.tracker.render_pipelines.insert_single(pipeline); + + pipeline.same_device_as(cmd_buf.as_ref())?; + + state + .info + .context + .check_compatible(&pipeline.pass_context, pipeline.as_ref()) + .map_err(RenderCommandError::IncompatiblePipelineTargets)?; + + state.pipeline_flags = pipeline.flags; + + if pipeline.flags.contains(PipelineFlags::WRITES_DEPTH) && state.info.is_depth_read_only { + return Err(RenderCommandError::IncompatibleDepthAccess(pipeline.error_ident()).into()); + } + if pipeline.flags.contains(PipelineFlags::WRITES_STENCIL) && state.info.is_stencil_read_only { + return Err(RenderCommandError::IncompatibleStencilAccess(pipeline.error_ident()).into()); + } + + state + .blend_constant + .require(pipeline.flags.contains(PipelineFlags::BLEND_CONSTANT)); + + unsafe { + state.raw_encoder.set_render_pipeline(pipeline.raw()); + } + + if pipeline.flags.contains(PipelineFlags::STENCIL_REFERENCE) { + unsafe { + state + .raw_encoder + .set_stencil_reference(state.stencil_reference); + } + } + + // Rebind resource + if state.binder.pipeline_layout.is_none() + || !state + .binder + .pipeline_layout + .as_ref() + .unwrap() + .is_equal(&pipeline.layout) + { + let (start_index, entries) = state + .binder + .change_pipeline_layout(&pipeline.layout, &pipeline.late_sized_buffer_groups); + if !entries.is_empty() { + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group.try_raw(state.snatch_guard)?; + unsafe { + state.raw_encoder.set_bind_group( + pipeline.layout.raw(), + start_index as u32 + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + + // Clear push constant ranges + let non_overlapping = + super::bind::compute_nonoverlapping_ranges(&pipeline.layout.push_constant_ranges); + for range in non_overlapping { + let offset = range.range.start; + let size_bytes = range.range.end - offset; + super::push_constant_clear(offset, size_bytes, |clear_offset, clear_data| unsafe { + state.raw_encoder.set_push_constants( + pipeline.layout.raw(), + range.stages, + clear_offset, + clear_data, + ); + }); + } + } + + // Initialize each `vertex.inputs[i].step` from + // `pipeline.vertex_steps[i]`. Enlarge `vertex.inputs` + // as necessary to accommodate all slots in the + // pipeline. If `vertex.inputs` is longer, fill the + // extra entries with default `VertexStep`s. + while state.vertex.inputs.len() < pipeline.vertex_steps.len() { + state.vertex.inputs.push(VertexBufferState::EMPTY); + } + + // This is worse as a `zip`, but it's close. + let mut steps = pipeline.vertex_steps.iter(); + for input in state.vertex.inputs.iter_mut() { + input.step = steps.next().cloned().unwrap_or_default(); + } + + // Update vertex buffer limits. + state.vertex.update_limits(); + Ok(()) +} + +fn set_index_buffer( + state: &mut State, + cmd_buf: &Arc>, + buffer: Arc>, + index_format: IndexFormat, + offset: u64, + size: Option, +) -> Result<(), RenderPassErrorInner> { + api_log!("RenderPass::set_index_buffer {}", buffer.error_ident()); + + state + .info + .usage_scope + .buffers + .merge_single(&buffer, hal::BufferUses::INDEX)?; + + buffer.same_device_as(cmd_buf.as_ref())?; + + buffer.check_usage(BufferUsages::INDEX)?; + let buf_raw = buffer.try_raw(state.snatch_guard)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + state.index.update_buffer(offset..end, index_format); + + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + &buffer, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + )); + + let bb = hal::BufferBinding { + buffer: buf_raw, + offset, + size, + }; + unsafe { + state.raw_encoder.set_index_buffer(bb, index_format); + } + Ok(()) +} + +fn set_vertex_buffer( + state: &mut State, + cmd_buf: &Arc>, + slot: u32, + buffer: Arc>, + offset: u64, + size: Option, +) -> Result<(), RenderPassErrorInner> { + api_log!( + "RenderPass::set_vertex_buffer {slot} {}", + buffer.error_ident() + ); + + state + .info + .usage_scope + .buffers + .merge_single(&buffer, hal::BufferUses::VERTEX)?; + + buffer.same_device_as(cmd_buf.as_ref())?; + + let max_vertex_buffers = state.device.limits.max_vertex_buffers; + if slot >= max_vertex_buffers { + return Err(RenderCommandError::VertexBufferIndexOutOfRange { + index: slot, + max: max_vertex_buffers, + } + .into()); + } + + buffer.check_usage(BufferUsages::VERTEX)?; + let buf_raw = buffer.try_raw(state.snatch_guard)?; + + let empty_slots = (1 + slot as usize).saturating_sub(state.vertex.inputs.len()); + state + .vertex + .inputs + .extend(iter::repeat(VertexBufferState::EMPTY).take(empty_slots)); + let vertex_state = &mut state.vertex.inputs[slot as usize]; + //TODO: where are we checking that the offset is in bound? + vertex_state.total_size = match size { + Some(s) => s.get(), + None => buffer.size - offset, + }; + vertex_state.bound = true; + + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + &buffer, + offset..(offset + vertex_state.total_size), + MemoryInitKind::NeedsInitializedMemory, + )); + + let bb = hal::BufferBinding { + buffer: buf_raw, + offset, + size, + }; + unsafe { + state.raw_encoder.set_vertex_buffer(slot, bb); + } + state.vertex.update_limits(); + Ok(()) +} + +fn set_blend_constant(state: &mut State, color: &Color) { + api_log!("RenderPass::set_blend_constant"); + + state.blend_constant = OptionalState::Set; + let array = [ + color.r as f32, + color.g as f32, + color.b as f32, + color.a as f32, + ]; + unsafe { + state.raw_encoder.set_blend_constants(&array); + } +} + +fn set_stencil_reference(state: &mut State, value: u32) { + api_log!("RenderPass::set_stencil_reference {value}"); + + state.stencil_reference = value; + if state + .pipeline_flags + .contains(PipelineFlags::STENCIL_REFERENCE) + { + unsafe { + state.raw_encoder.set_stencil_reference(value); + } + } +} + +fn set_viewport( + state: &mut State, + rect: Rect, + depth_min: f32, + depth_max: f32, +) -> Result<(), RenderPassErrorInner> { + api_log!("RenderPass::set_viewport {rect:?}"); + if rect.x < 0.0 + || rect.y < 0.0 + || rect.w <= 0.0 + || rect.h <= 0.0 + || rect.x + rect.w > state.info.extent.width as f32 + || rect.y + rect.h > state.info.extent.height as f32 + { + return Err(RenderCommandError::InvalidViewportRect(rect, state.info.extent).into()); + } + if !(0.0..=1.0).contains(&depth_min) || !(0.0..=1.0).contains(&depth_max) { + return Err(RenderCommandError::InvalidViewportDepth(depth_min, depth_max).into()); + } + let r = hal::Rect { + x: rect.x, + y: rect.y, + w: rect.w, + h: rect.h, + }; + unsafe { + state.raw_encoder.set_viewport(&r, depth_min..depth_max); + } + Ok(()) +} + +fn set_push_constant( + state: &mut State, + push_constant_data: &[u32], + stages: ShaderStages, + offset: u32, + size_bytes: u32, + values_offset: Option, +) -> Result<(), RenderPassErrorInner> { + api_log!("RenderPass::set_push_constants"); + + let values_offset = values_offset.ok_or(RenderPassErrorInner::InvalidValuesOffset)?; + + let end_offset_bytes = offset + size_bytes; + let values_end_offset = (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = &push_constant_data[(values_offset as usize)..values_end_offset]; + + let pipeline_layout = state + .binder + .pipeline_layout + .as_ref() + .ok_or(DrawError::MissingPipeline)?; + + pipeline_layout + .validate_push_constant_ranges(stages, offset, end_offset_bytes) + .map_err(RenderCommandError::from)?; + + unsafe { + state + .raw_encoder + .set_push_constants(pipeline_layout.raw(), stages, offset, data_slice) + } + Ok(()) +} + +fn set_scissor( + state: &mut State, + rect: Rect, +) -> Result<(), RenderPassErrorInner> { + api_log!("RenderPass::set_scissor_rect {rect:?}"); + + if rect.x + rect.w > state.info.extent.width || rect.y + rect.h > state.info.extent.height { + return Err(RenderCommandError::InvalidScissorRect(rect, state.info.extent).into()); + } + let r = hal::Rect { + x: rect.x, + y: rect.y, + w: rect.w, + h: rect.h, + }; + unsafe { + state.raw_encoder.set_scissor_rect(&r); + } + Ok(()) +} + +fn draw( + state: &mut State, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, +) -> Result<(), DrawError> { + api_log!("RenderPass::draw {vertex_count} {instance_count} {first_vertex} {first_instance}"); + + state.is_ready(false)?; + + let last_vertex = first_vertex as u64 + vertex_count as u64; + let vertex_limit = state.vertex.vertex_limit; + if last_vertex > vertex_limit { + return Err(DrawError::VertexBeyondLimit { + last_vertex, + vertex_limit, + slot: state.vertex.vertex_limit_slot, + }); + } + let last_instance = first_instance as u64 + instance_count as u64; + let instance_limit = state.vertex.instance_limit; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + slot: state.vertex.instance_limit_slot, + }); + } + + unsafe { + if instance_count > 0 && vertex_count > 0 { + state + .raw_encoder + .draw(first_vertex, vertex_count, first_instance, instance_count); + } + } + Ok(()) +} + +fn draw_indexed( + state: &mut State, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, +) -> Result<(), DrawError> { + api_log!("RenderPass::draw_indexed {index_count} {instance_count} {first_index} {base_vertex} {first_instance}"); + + state.is_ready(true)?; + + let last_index = first_index as u64 + index_count as u64; + let index_limit = state.index.limit; + if last_index > index_limit { + return Err(DrawError::IndexBeyondLimit { + last_index, + index_limit, + }); + } + let last_instance = first_instance as u64 + instance_count as u64; + let instance_limit = state.vertex.instance_limit; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + slot: state.vertex.instance_limit_slot, + }); + } + + unsafe { + if instance_count > 0 && index_count > 0 { + state.raw_encoder.draw_indexed( + first_index, + index_count, + base_vertex, + first_instance, + instance_count, + ); + } + } + Ok(()) +} + +fn multi_draw_indirect( + state: &mut State, + indirect_buffer: Arc>, + offset: u64, + count: Option, + indexed: bool, +) -> Result<(), RenderPassErrorInner> { + api_log!( + "RenderPass::draw_indirect (indexed:{indexed}) {} {offset} {count:?}", + indirect_buffer.error_ident() + ); + + state.is_ready(indexed)?; + + let stride = match indexed { + false => mem::size_of::(), + true => mem::size_of::(), + }; + + if count.is_some() { + state + .device + .require_features(wgt::Features::MULTI_DRAW_INDIRECT)?; + } + state + .device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?; + + state + .info + .usage_scope + .buffers + .merge_single(&indirect_buffer, hal::BufferUses::INDIRECT)?; + + indirect_buffer.check_usage(BufferUsages::INDIRECT)?; + let indirect_raw = indirect_buffer.try_raw(state.snatch_guard)?; + + let actual_count = count.map_or(1, |c| c.get()); + + let end_offset = offset + stride as u64 * actual_count as u64; + if end_offset > indirect_buffer.size { + return Err(RenderPassErrorInner::IndirectBufferOverrun { + count, + offset, + end_offset, + buffer_size: indirect_buffer.size, + }); + } + + state.buffer_memory_init_actions.extend( + indirect_buffer.initialization_status.read().create_action( + &indirect_buffer, + offset..end_offset, + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + match indexed { + false => unsafe { + state + .raw_encoder + .draw_indirect(indirect_raw, offset, actual_count); + }, + true => unsafe { + state + .raw_encoder + .draw_indexed_indirect(indirect_raw, offset, actual_count); + }, + } + Ok(()) +} + +fn multi_draw_indirect_count( + state: &mut State, + indirect_buffer: Arc>, + offset: u64, + count_buffer: Arc>, + count_buffer_offset: u64, + max_count: u32, + indexed: bool, +) -> Result<(), RenderPassErrorInner> { + api_log!( + "RenderPass::multi_draw_indirect_count (indexed:{indexed}) {} {offset} {} {count_buffer_offset:?} {max_count:?}", + indirect_buffer.error_ident(), + count_buffer.error_ident() + ); + + state.is_ready(indexed)?; + + let stride = match indexed { + false => mem::size_of::(), + true => mem::size_of::(), + } as u64; + + state + .device + .require_features(wgt::Features::MULTI_DRAW_INDIRECT_COUNT)?; + state + .device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?; + + state + .info + .usage_scope + .buffers + .merge_single(&indirect_buffer, hal::BufferUses::INDIRECT)?; + + indirect_buffer.check_usage(BufferUsages::INDIRECT)?; + let indirect_raw = indirect_buffer.try_raw(state.snatch_guard)?; + + state + .info + .usage_scope + .buffers + .merge_single(&count_buffer, hal::BufferUses::INDIRECT)?; + + count_buffer.check_usage(BufferUsages::INDIRECT)?; + let count_raw = count_buffer.try_raw(state.snatch_guard)?; + + let end_offset = offset + stride * max_count as u64; + if end_offset > indirect_buffer.size { + return Err(RenderPassErrorInner::IndirectBufferOverrun { + count: None, + offset, + end_offset, + buffer_size: indirect_buffer.size, + }); + } + state.buffer_memory_init_actions.extend( + indirect_buffer.initialization_status.read().create_action( + &indirect_buffer, + offset..end_offset, + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + let begin_count_offset = count_buffer_offset; + let end_count_offset = count_buffer_offset + 4; + if end_count_offset > count_buffer.size { + return Err(RenderPassErrorInner::IndirectCountBufferOverrun { + begin_count_offset, + end_count_offset, + count_buffer_size: count_buffer.size, + }); + } + state.buffer_memory_init_actions.extend( + count_buffer.initialization_status.read().create_action( + &count_buffer, + count_buffer_offset..end_count_offset, + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + match indexed { + false => unsafe { + state.raw_encoder.draw_indirect_count( + indirect_raw, + offset, + count_raw, + count_buffer_offset, + max_count, + ); + }, + true => unsafe { + state.raw_encoder.draw_indexed_indirect_count( + indirect_raw, + offset, + count_raw, + count_buffer_offset, + max_count, + ); + }, + } + Ok(()) +} + +fn push_debug_group(state: &mut State, string_data: &[u8], len: usize) { + state.debug_scope_depth += 1; + if !state + .device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + let label = + str::from_utf8(&string_data[state.string_offset..state.string_offset + len]).unwrap(); + + api_log!("RenderPass::push_debug_group {label:?}"); + unsafe { + state.raw_encoder.begin_debug_marker(label); + } + } + state.string_offset += len; +} + +fn pop_debug_group(state: &mut State) -> Result<(), RenderPassErrorInner> { + api_log!("RenderPass::pop_debug_group"); + + if state.debug_scope_depth == 0 { + return Err(RenderPassErrorInner::InvalidPopDebugGroup); + } + state.debug_scope_depth -= 1; + if !state + .device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + unsafe { + state.raw_encoder.end_debug_marker(); + } + } + Ok(()) +} + +fn insert_debug_marker(state: &mut State, string_data: &[u8], len: usize) { + if !state + .device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + let label = + str::from_utf8(&string_data[state.string_offset..state.string_offset + len]).unwrap(); + api_log!("RenderPass::insert_debug_marker {label:?}"); + unsafe { + state.raw_encoder.insert_debug_marker(label); + } + } + state.string_offset += len; +} + +fn write_timestamp( + state: &mut State, + pending_query_resets: &mut QueryResetMap, + query_set: Arc>, + query_index: u32, +) -> Result<(), RenderPassErrorInner> { + api_log!( + "RenderPass::write_timestamps {query_index} {}", + query_set.error_ident() + ); + + state + .device + .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES)?; + + let query_set = state.tracker.query_sets.insert_single(query_set); + + query_set.validate_and_write_timestamp( + state.raw_encoder, + query_index, + Some(pending_query_resets), + )?; + Ok(()) +} + +fn execute_bundle( + state: &mut State, + cmd_buf: &Arc>, + bundle: Arc>, +) -> Result<(), RenderPassErrorInner> { + api_log!("RenderPass::execute_bundle {}", bundle.error_ident()); + + // Have to clone the bundle arc, otherwise we keep a mutable reference to the bundle + // while later trying to add the bundle's resources to the tracker. + let bundle = state.tracker.bundles.insert_single(bundle).clone(); + + bundle.same_device_as(cmd_buf.as_ref())?; + + state + .info + .context + .check_compatible(&bundle.context, bundle.as_ref()) + .map_err(RenderPassErrorInner::IncompatibleBundleTargets)?; + + if (state.info.is_depth_read_only && !bundle.is_depth_read_only) + || (state.info.is_stencil_read_only && !bundle.is_stencil_read_only) + { + return Err( + RenderPassErrorInner::IncompatibleBundleReadOnlyDepthStencil { + pass_depth: state.info.is_depth_read_only, + pass_stencil: state.info.is_stencil_read_only, + bundle_depth: bundle.is_depth_read_only, + bundle_stencil: bundle.is_stencil_read_only, + }, + ); + } + + state + .buffer_memory_init_actions + .extend( + bundle + .buffer_memory_init_actions + .iter() + .filter_map(|action| { + action + .buffer + .initialization_status + .read() + .check_action(action) + }), + ); + for action in bundle.texture_memory_init_actions.iter() { + state + .info + .pending_discard_init_fixups + .extend(state.texture_memory_actions.register_init_action(action)); + } + + unsafe { bundle.execute(state.raw_encoder, state.snatch_guard) }.map_err(|e| match e { + ExecutionError::DestroyedResource(e) => RenderCommandError::DestroyedResource(e), + ExecutionError::Unimplemented(what) => RenderCommandError::Unimplemented(what), + })?; + + unsafe { + state.info.usage_scope.merge_render_bundle(&bundle.used)?; + state.tracker.add_from_render_bundle(&bundle.used)?; + }; + state.reset_bundle(); + Ok(()) +} + impl Global { pub fn render_pass_set_bind_group( &self, pass: &mut RenderPass, index: u32, bind_group_id: id::BindGroupId, - offsets: &[wgt::DynamicOffset], + offsets: &[DynamicOffset], ) -> Result<(), RenderPassError> { - let scope = PassErrorScope::SetBindGroup(bind_group_id); + let scope = PassErrorScope::SetBindGroup; let base = pass .base .as_mut() @@ -2497,7 +2668,7 @@ impl Global { pass: &mut RenderPass, pipeline_id: id::RenderPipelineId, ) -> Result<(), RenderPassError> { - let scope = PassErrorScope::SetPipelineRender(pipeline_id); + let scope = PassErrorScope::SetPipelineRender; let redundant = pass.current_pipeline.set_and_check_redundant(pipeline_id); let base = pass.base_mut(scope)?; @@ -2520,7 +2691,7 @@ impl Global { offset: BufferAddress, size: Option, ) -> Result<(), RenderPassError> { - let scope = PassErrorScope::SetIndexBuffer(buffer_id); + let scope = PassErrorScope::SetIndexBuffer; let base = pass.base_mut(scope)?; base.commands.push(RenderCommand::SetIndexBuffer { @@ -2541,7 +2712,7 @@ impl Global { offset: BufferAddress, size: Option, ) -> Result<(), RenderPassError> { - let scope = PassErrorScope::SetVertexBuffer(buffer_id); + let scope = PassErrorScope::SetVertexBuffer; let base = pass.base_mut(scope)?; base.commands.push(RenderCommand::SetVertexBuffer { @@ -2623,7 +2794,7 @@ impl Global { pub fn render_pass_set_push_constants( &self, pass: &mut RenderPass, - stages: wgt::ShaderStages, + stages: ShaderStages, offset: u32, data: &[u8], ) -> Result<(), RenderPassError> { @@ -2670,7 +2841,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::Draw, indexed: false, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; @@ -2696,7 +2866,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::Draw, indexed: true, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; @@ -2720,7 +2889,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::DrawIndirect, indexed: false, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; @@ -2743,7 +2911,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::DrawIndirect, indexed: true, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; @@ -2767,7 +2934,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::MultiDrawIndirect, indexed: false, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; @@ -2791,7 +2957,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::MultiDrawIndirect, indexed: true, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; @@ -2817,7 +2982,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::MultiDrawIndirectCount, indexed: false, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; @@ -2845,7 +3009,6 @@ impl Global { let scope = PassErrorScope::Draw { kind: DrawKind::MultiDrawIndirectCount, indexed: true, - pipeline: pass.current_pipeline.last_state, }; let base = pass.base_mut(scope)?; diff --git a/wgpu-core/src/command/render_command.rs b/wgpu-core/src/command/render_command.rs index 22072bfc6e..3140b78e68 100644 --- a/wgpu-core/src/command/render_command.rs +++ b/wgpu-core/src/command/render_command.rs @@ -153,7 +153,7 @@ impl RenderCommand { num_dynamic_offsets, bind_group: bind_group_guard.get_owned(bind_group_id).map_err(|_| { RenderPassError { - scope: PassErrorScope::SetBindGroup(bind_group_id), + scope: PassErrorScope::SetBindGroup, inner: RenderPassErrorInner::InvalidBindGroup(index), } })?, @@ -163,7 +163,7 @@ impl RenderCommand { pipelines_guard .get_owned(pipeline_id) .map_err(|_| RenderPassError { - scope: PassErrorScope::SetPipelineRender(pipeline_id), + scope: PassErrorScope::SetPipelineRender, inner: RenderCommandError::InvalidPipeline(pipeline_id).into(), })?, ), @@ -228,7 +228,7 @@ impl RenderCommand { } => ArcRenderCommand::SetIndexBuffer { buffer: buffers_guard.get_owned(buffer_id).map_err(|_| { RenderPassError { - scope: PassErrorScope::SetIndexBuffer(buffer_id), + scope: PassErrorScope::SetIndexBuffer, inner: RenderCommandError::InvalidBufferId(buffer_id).into(), } })?, @@ -246,7 +246,7 @@ impl RenderCommand { slot, buffer: buffers_guard.get_owned(buffer_id).map_err(|_| { RenderPassError { - scope: PassErrorScope::SetVertexBuffer(buffer_id), + scope: PassErrorScope::SetVertexBuffer, inner: RenderCommandError::InvalidBufferId(buffer_id).into(), } })?, @@ -315,7 +315,6 @@ impl RenderCommand { DrawKind::DrawIndirect }, indexed, - pipeline: None, }, inner: RenderCommandError::InvalidBufferId(buffer_id).into(), } @@ -336,7 +335,6 @@ impl RenderCommand { let scope = PassErrorScope::Draw { kind: DrawKind::MultiDrawIndirectCount, indexed, - pipeline: None, }; ArcRenderCommand::MultiDrawIndirectCount { buffer: buffers_guard.get_owned(buffer_id).map_err(|_| { diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 855cea1b42..ab774d87c0 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -4,7 +4,8 @@ use crate::{ hub::Hub, id::{BindGroupLayoutId, PipelineLayoutId}, resource::{ - Buffer, BufferAccessError, BufferAccessResult, BufferMapOperation, ResourceErrorIdent, + Buffer, BufferAccessError, BufferAccessResult, BufferMapOperation, Resource, + ResourceErrorIdent, }, snatch::SnatchGuard, Label, DOWNLEVEL_ERROR_MESSAGE, @@ -69,12 +70,6 @@ impl AttachmentData { } } -#[derive(Debug, Copy, Clone)] -pub enum RenderPassCompatibilityCheckType { - RenderPipeline, - RenderBundle, -} - #[derive(Clone, Debug, Hash, PartialEq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub(crate) struct RenderPassContext { @@ -86,44 +81,44 @@ pub(crate) struct RenderPassContext { #[non_exhaustive] pub enum RenderPassCompatibilityError { #[error( - "Incompatible color attachments at indices {indices:?}: the RenderPass uses textures with formats {expected:?} but the {ty:?} uses attachments with formats {actual:?}", + "Incompatible color attachments at indices {indices:?}: the RenderPass uses textures with formats {expected:?} but the {res} uses attachments with formats {actual:?}", )] IncompatibleColorAttachment { indices: Vec, expected: Vec>, actual: Vec>, - ty: RenderPassCompatibilityCheckType, + res: ResourceErrorIdent, }, #[error( - "Incompatible depth-stencil attachment format: the RenderPass uses a texture with format {expected:?} but the {ty:?} uses an attachment with format {actual:?}", + "Incompatible depth-stencil attachment format: the RenderPass uses a texture with format {expected:?} but the {res} uses an attachment with format {actual:?}", )] IncompatibleDepthStencilAttachment { expected: Option, actual: Option, - ty: RenderPassCompatibilityCheckType, + res: ResourceErrorIdent, }, #[error( - "Incompatible sample count: the RenderPass uses textures with sample count {expected:?} but the {ty:?} uses attachments with format {actual:?}", + "Incompatible sample count: the RenderPass uses textures with sample count {expected:?} but the {res} uses attachments with format {actual:?}", )] IncompatibleSampleCount { expected: u32, actual: u32, - ty: RenderPassCompatibilityCheckType, + res: ResourceErrorIdent, }, - #[error("Incompatible multiview setting: the RenderPass uses setting {expected:?} but the {ty:?} uses setting {actual:?}")] + #[error("Incompatible multiview setting: the RenderPass uses setting {expected:?} but the {res} uses setting {actual:?}")] IncompatibleMultiview { expected: Option, actual: Option, - ty: RenderPassCompatibilityCheckType, + res: ResourceErrorIdent, }, } impl RenderPassContext { // Assumes the renderpass only contains one subpass - pub(crate) fn check_compatible( + pub(crate) fn check_compatible( &self, other: &Self, - ty: RenderPassCompatibilityCheckType, + res: &T, ) -> Result<(), RenderPassCompatibilityError> { if self.attachments.colors != other.attachments.colors { let indices = self @@ -138,7 +133,7 @@ impl RenderPassContext { indices, expected: self.attachments.colors.iter().cloned().collect(), actual: other.attachments.colors.iter().cloned().collect(), - ty, + res: res.error_ident(), }); } if self.attachments.depth_stencil != other.attachments.depth_stencil { @@ -146,7 +141,7 @@ impl RenderPassContext { RenderPassCompatibilityError::IncompatibleDepthStencilAttachment { expected: self.attachments.depth_stencil, actual: other.attachments.depth_stencil, - ty, + res: res.error_ident(), }, ); } @@ -154,14 +149,14 @@ impl RenderPassContext { return Err(RenderPassCompatibilityError::IncompatibleSampleCount { expected: self.sample_count, actual: other.sample_count, - ty, + res: res.error_ident(), }); } if self.multiview != other.multiview { return Err(RenderPassCompatibilityError::IncompatibleMultiview { expected: self.multiview, actual: other.multiview, - ty, + res: res.error_ident(), }); } Ok(())