diff --git a/Cargo.toml b/Cargo.toml index fdec29b8..4b2d5927 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,9 @@ bevy = { git = "https://github.com/processing/bevy", branch = "main", default-fe "shader_format_wesl", "free_camera", "pan_camera", - + + # image formats Processing loads + "jpeg", ] } bevy_naga_reflect = { git = "https://github.com/tychedelia/bevy_naga_reflect" } bevy_cuda = { git = "https://github.com/tychedelia/bevy_cuda" } diff --git a/crates/processing_cuda/src/lib.rs b/crates/processing_cuda/src/lib.rs index 8f970b99..c9520b53 100644 --- a/crates/processing_cuda/src/lib.rs +++ b/crates/processing_cuda/src/lib.rs @@ -7,8 +7,8 @@ use bevy::render::renderer::RenderDevice; use bevy_cuda::{CudaBuffer, CudaContext}; use processing_core::app_mut; use processing_core::error::{ProcessingError, Result}; -use processing_render::graphics::view_target; -use processing_render::image::{Image, gpu_image, pixel_size}; +use processing_render::graphics::{invalidate_pixels, main_texture, view_target}; +use processing_render::image::{Image, pixel_size, texture_for_read, texture_for_write}; #[derive(Component)] pub struct CudaImageBuffer { @@ -44,9 +44,18 @@ fn cuda_ctx(world: &World) -> Result<&CudaContext> { .ok_or(ProcessingError::CudaError("CUDA not available".into())) } -fn resolve_texture(app: &mut App, entity: Entity) -> Result<(Texture, TextureFormat, u32, u32)> { +/// The texture to copy from, or with `write` to copy into, which leaves an image's CPU copy stale +fn resolve_texture( + app: &mut App, + entity: Entity, + write: bool, +) -> Result<(Texture, TextureFormat, u32, u32)> { if app.world().get::(entity).is_some() { - let texture = gpu_image(app, entity)?.texture.clone(); + let texture = if write { + texture_for_write(app, entity)? + } else { + texture_for_read(app, entity)? + }; let p_image = app.world().get::(entity).unwrap(); return Ok(( texture, @@ -55,9 +64,11 @@ fn resolve_texture(app: &mut App, entity: Entity) -> Result<(Texture, TextureFor p_image.size.height, )); } - if let Ok(vt) = view_target(app, entity) { - let texture = vt.main_texture().clone(); - let fmt = vt.main_texture_format(); + if let Ok(texture) = main_texture(app, entity) { + if write { + invalidate_pixels(app, entity); + } + let fmt = view_target(app, entity)?.main_texture_format(); let size = texture.size(); return Ok((texture, fmt, size.width, size.height)); } @@ -66,7 +77,7 @@ fn resolve_texture(app: &mut App, entity: Entity) -> Result<(Texture, TextureFor pub fn cuda_export(entity: Entity) -> Result<()> { app_mut(|app| { - let (texture, texture_format, width, height) = resolve_texture(app, entity)?; + let (texture, texture_format, width, height) = resolve_texture(app, entity, false)?; let px_size = pixel_size(texture_format)?; let buffer_size = (width as u64) * (height as u64) * (px_size as u64); @@ -103,7 +114,7 @@ pub fn cuda_export(entity: Entity) -> Result<()> { pub fn cuda_import(entity: Entity, src_device_ptr: u64, byte_size: u64) -> Result<()> { app_mut(|app| { - let (texture, texture_format, width, height) = resolve_texture(app, entity)?; + let (texture, texture_format, width, height) = resolve_texture(app, entity, true)?; let existing = app.world().get::(entity); let needs_alloc = existing.is_none_or(|buf| buf.buffer.size() != byte_size); @@ -147,7 +158,7 @@ pub fn cuda_import(entity: Entity, src_device_ptr: u64, byte_size: u64) -> Resul pub fn cuda_write_back(entity: Entity) -> Result<()> { app_mut(|app| { - let (texture, _, _, _) = resolve_texture(app, entity)?; + let (texture, _, _, _) = resolve_texture(app, entity, true)?; let cuda_buf = app .world() diff --git a/crates/processing_ffi/src/lib.rs b/crates/processing_ffi/src/lib.rs index f78a8660..ed1bb0ac 100644 --- a/crates/processing_ffi/src/lib.rs +++ b/crates/processing_ffi/src/lib.rs @@ -1763,7 +1763,7 @@ pub unsafe extern "C" fn processing_graphics_update_region( .iter() .map(|color| color.to_linear()) .collect(); - graphics_update_region(graphics_entity, x, y, width, height, &pixels) + graphics_update_region(graphics_entity, x.into(), y.into(), width, height, &pixels) }); } @@ -1776,7 +1776,16 @@ pub unsafe extern "C" fn processing_graphics_update_region( pub extern "C" fn processing_graphics_set(graphics_id: u64, x: u32, y: u32, color: Color) { error::clear_error(); let graphics_entity = Entity::from_bits(graphics_id); - error::check(|| graphics_update_region(graphics_entity, x, y, 1, 1, &[color.to_linear()])); + error::check(|| { + graphics_update_region( + graphics_entity, + x.into(), + y.into(), + 1, + 1, + &[color.to_linear()], + ) + }); } /// Set the tint color applied to images. diff --git a/crates/processing_node/src/lib.rs b/crates/processing_node/src/lib.rs index 95e82cba..7fb6a8d7 100644 --- a/crates/processing_node/src/lib.rs +++ b/crates/processing_node/src/lib.rs @@ -1202,8 +1202,8 @@ pub fn js_graphics_update_region( ) -> Result<()> { check(graphics_update_region( ent(graphics_id), - x, - y, + x.into(), + y.into(), width, height, &floats_to_linear(&pixels, space), @@ -1224,8 +1224,8 @@ pub fn js_graphics_set( let color = Color::new(c1, c2, c3, a, space).to_linear(); check(graphics_update_region( ent(graphics_id), - x, - y, + x.into(), + y.into(), 1, 1, &[color], diff --git a/crates/processing_pyo3/src/graphics.rs b/crates/processing_pyo3/src/graphics.rs index 591fd911..395b9f1f 100644 --- a/crates/processing_pyo3/src/graphics.rs +++ b/crates/processing_pyo3/src/graphics.rs @@ -1,4 +1,4 @@ -use crate::color::{ColorMode, extract_color_with_mode}; +use crate::color::{ColorMode, ColorSpace, extract_color_with_mode, parse_numeric}; use crate::glfw::GlfwContext; use crate::input; use crate::math::{extract_vec2, extract_vec3, extract_vec4}; @@ -37,10 +37,14 @@ fn flatten_floats(args: &Bound<'_, PyTuple>) -> PyResult> { fn pixels_to_pylist(py: Python<'_>, pixels: &[LinearRgba]) -> PyResult> { let colors = pixels .iter() - .map(|p| crate::color::PyColor(bevy::prelude::Color::LinearRgba(*p))); + .map(|p| crate::color::PyColor(bevy::prelude::Color::Srgba(Srgba::from(*p)))); Ok(PyList::new(py, colors)?.unbind()) } +fn pixel_item_error() -> PyErr { + PyValueError::new_err("pixels must contain Color values or [r, g, b, (a)] sequences") +} + /// Convert a Python sequence of colors (as returned by `load_pixels()`, or /// individual `Color`/`[r,g,b,a]` values) into linear-RGBA pixels. fn pyseq_to_pixels(seq: &Bound<'_, PyAny>) -> PyResult> { @@ -49,14 +53,21 @@ fn pyseq_to_pixels(seq: &Bound<'_, PyAny>) -> PyResult> { let item = item?; let color = if let Ok(c) = item.extract::() { c.0 - } else if let Ok(rgba) = item.extract::<[f32; 4]>() { - bevy::prelude::Color::LinearRgba(LinearRgba::new(rgba[0], rgba[1], rgba[2], rgba[3])) - } else if let Ok(rgb) = item.extract::<[f32; 3]>() { - bevy::prelude::Color::LinearRgba(LinearRgba::new(rgb[0], rgb[1], rgb[2], 1.0)) } else { - return Err(PyValueError::new_err( - "pixels must contain Color values or [r, g, b, (a)] sequences", - )); + // bare components follow the default color mode: ints 0-255, floats 0-1 + let space = ColorSpace::Srgb; + let parts: Vec> = item + .try_iter() + .and_then(|it| it.collect()) + .map_err(|_| pixel_item_error())?; + if parts.len() != 3 && parts.len() != 4 { + return Err(pixel_item_error()); + } + let mut c = [1.0f32; 4]; + for (ch, part) in parts.iter().enumerate() { + c[ch] = parse_numeric(&space, part, ch).map_err(|_| pixel_item_error())?; + } + space.color(c[0], c[1], c[2], c[3]) }; out.push(LinearRgba::from(color)); } @@ -547,6 +558,17 @@ impl<'a, 'py> FromPyObject<'a, 'py> for ImageRef { if let Ok(img) = ob.extract::>() { return Ok(ImageRef { entity: img.entity }); } + // an offscreen Graphics renders into an image; a window has no texture to sample + if let Ok(g) = ob.extract::>() { + if image_size(g.surface.entity).is_err() { + return Err(pyo3::exceptions::PyTypeError::new_err( + "a window Graphics cannot be drawn as an image", + )); + } + return Ok(ImageRef { + entity: g.surface.entity, + }); + } #[cfg(feature = "webcam")] if let Ok(cam) = ob.extract::>() { return Ok(ImageRef { @@ -554,7 +576,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for ImageRef { }); } Err(pyo3::exceptions::PyTypeError::new_err( - "expected an Image or Webcam", + "expected an Image, Graphics or Webcam", )) } } @@ -605,25 +627,19 @@ impl Image { .map_err(|e| PyRuntimeError::new_err(format!("{e}"))) } - /// Read a single pixel as a `Color` (Processing `get`). - fn get(&self, x: u32, y: u32) -> PyResult { - let (w, h) = - image_size(self.entity).map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; - if x >= w || y >= h { - return Err(PyValueError::new_err(format!( - "pixel ({x}, {y}) out of bounds for {w}x{h} image" - ))); - } - let pixels = - image_readback(self.entity).map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; - let p = pixels[(y * w + x) as usize]; - Ok(crate::color::PyColor(bevy::prelude::Color::LinearRgba(p))) + /// Read a single pixel as a `Color` (Processing `get`); outside the image it is transparent black. + fn get(&self, x: i64, y: i64) -> PyResult { + let p = image_get_pixel(self.entity, x, y) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + Ok(crate::color::PyColor(bevy::prelude::Color::Srgba( + p.map_or(Srgba::NONE, Srgba::from), + ))) } - /// Write a single pixel (Processing `set`). Accepts a `Color`, hex string, - /// or color components in the default (sRGB) color mode. + /// Write a single pixel (Processing `set`); outside the image it is dropped. Accepts a + /// `Color`, hex string, or color components in the default (sRGB) color mode. #[pyo3(signature = (x, y, *args))] - fn set(&self, x: u32, y: u32, args: &Bound<'_, PyTuple>) -> PyResult<()> { + fn set(&self, x: i64, y: i64, args: &Bound<'_, PyTuple>) -> PyResult<()> { let color = extract_color_with_mode(args, &ColorMode::default())?; image_update_region(self.entity, x, y, 1, 1, &[LinearRgba::from(color)]) .map_err(|e| PyRuntimeError::new_err(format!("{e}"))) @@ -1030,30 +1046,23 @@ impl Graphics { } /// Read a single pixel as a `Color` (Processing `get`). - pub fn get(&self, x: u32, y: u32) -> PyResult { - if x >= self.width || y >= self.height { - return Err(PyValueError::new_err(format!( - "pixel ({x}, {y}) out of bounds for {}x{} canvas", - self.width, self.height - ))); - } - let pixels = - graphics_readback(self.entity).map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; - let p = pixels - .get((y * self.width + x) as usize) - .ok_or_else(|| PyValueError::new_err("pixel out of bounds"))?; - Ok(crate::color::PyColor(bevy::prelude::Color::LinearRgba(*p))) + pub fn get(&self, x: i64, y: i64) -> PyResult { + let p = graphics_get_pixel(self.entity, x, y) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + Ok(crate::color::PyColor(bevy::prelude::Color::Srgba( + p.map_or(Srgba::NONE, Srgba::from), + ))) } - /// Write a single pixel (Processing `set`). + /// Write a single pixel (Processing `set`); outside the canvas it is dropped. #[pyo3(signature = (x, y, *args))] - pub fn set(&self, x: u32, y: u32, args: &Bound<'_, PyTuple>) -> PyResult<()> { + pub fn set(&self, x: i64, y: i64, args: &Bound<'_, PyTuple>) -> PyResult<()> { let color = extract_color_with_mode( args, &graphics_get_color_mode(self.entity) .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?, )?; - graphics_update_region(self.entity, x, y, 1, 1, &[LinearRgba::from(color)]) + graphics_set_pixel(self.entity, x, y, LinearRgba::from(color)) .map_err(|e| PyRuntimeError::new_err(format!("{e}"))) } diff --git a/crates/processing_pyo3/src/lib.rs b/crates/processing_pyo3/src/lib.rs index 986c7b48..51f0d6ae 100644 --- a/crates/processing_pyo3/src/lib.rs +++ b/crates/processing_pyo3/src/lib.rs @@ -2320,7 +2320,7 @@ pub mod processing_py { #[pyfunction] #[pyo3(pass_module)] - fn get(module: &Bound<'_, PyModule>, x: u32, y: u32) -> PyResult { + fn get(module: &Bound<'_, PyModule>, x: i64, y: i64) -> PyResult { graphics!(module).get(x, y) } @@ -2328,8 +2328,8 @@ pub mod processing_py { #[pyo3(pass_module, signature = (x, y, *args))] fn set( module: &Bound<'_, PyModule>, - x: u32, - y: u32, + x: i64, + y: i64, args: &Bound<'_, PyTuple>, ) -> PyResult<()> { graphics!(module).set(x, y, args) diff --git a/crates/processing_render/src/ci.rs b/crates/processing_render/src/ci.rs index c929c39e..5acdeba6 100644 --- a/crates/processing_render/src/ci.rs +++ b/crates/processing_render/src/ci.rs @@ -125,13 +125,7 @@ pub fn done() -> bool { } pub(crate) fn readback_srgba8(app: &mut App, entity: Entity) -> Result<(u32, u32, Vec)> { - crate::graphics::flush(app, entity)?; - let vt = crate::graphics::view_target(app, entity)?; - let texture = vt.main_texture().clone(); - let raw = app - .world_mut() - .run_system_cached_with(crate::graphics::readback_raw, (entity, texture)) - .unwrap()?; + let raw = crate::graphics::read_pixels(app, entity)?; let rgba = match raw.format { TextureFormat::Rgba8UnormSrgb => raw.bytes, format => { diff --git a/crates/processing_render/src/graphics.rs b/crates/processing_render/src/graphics.rs index d2007a08..b157d58d 100644 --- a/crates/processing_render/src/graphics.rs +++ b/crates/processing_render/src/graphics.rs @@ -54,6 +54,10 @@ pub struct Graphics { readback_buffer: bevy::render::render_resource::Buffer, pub texture_format: TextureFormat, pub size: Extent3d, + /// CPU copy of the canvas, tightly packed; dropped whenever something draws + pixels: Option>, + /// Part of `pixels` written by `set` and not uploaded yet + dirty: Option, } pub fn view_target(app: &mut App, entity: Entity) -> Result<&ViewTarget> { @@ -279,6 +283,8 @@ pub fn create( readback_buffer, texture_format, size, + pixels: None, + dirty: None, }, )); @@ -332,6 +338,8 @@ pub fn sync_to_surface( height: physical_h, depth_or_array_layers: 1, }; + graphics.pixels = None; + graphics.dirty = None; graphics.readback_buffer = create_readback_buffer( &render_device, physical_w, @@ -628,6 +636,13 @@ pub fn screen_point( } pub fn flush(app: &mut App, entity: Entity) -> Result<()> { + // `set` pixels go up first so the draws queued after them land on top + upload_dirty(app, entity)?; + if has_commands(app, entity) + && let Some(mut graphics) = app.world_mut().get_mut::(entity) + { + graphics.pixels = None; + } graphics_mut!(app, entity).insert(Flush); // a mid-frame `background()` leaves the rest of the commands for another pass loop { @@ -643,6 +658,220 @@ pub fn flush(app: &mut App, entity: Entity) -> Result<()> { Ok(()) } +fn has_commands(app: &App, entity: Entity) -> bool { + app.world() + .get::(entity) + .is_some_and(|buffer| !buffer.commands.is_empty()) +} + +/// Uploads the canvas pixels `set` wrote since the last upload +fn upload_dirty(app: &mut App, entity: Entity) -> Result<()> { + let Some(graphics) = app.world().get::(entity) else { + return Ok(()); + }; + let Some(rect) = graphics.dirty else { + return Ok(()); + }; + let (size, format) = (graphics.size, graphics.texture_format); + let texture = view_target(app, entity)?.main_texture().clone(); + let world = app.world(); + if let Some(data) = world + .get::(entity) + .and_then(|g| g.pixels.as_deref()) + { + crate::image::write_rect( + world.resource::(), + &texture, + data, + size, + format, + rect, + )?; + } + if let Some(mut graphics) = app.world_mut().get_mut::(entity) { + graphics.dirty = None; + } + Ok(()) +} + +/// The canvas texture for a GPU reader, with pending `set` pixels uploaded +pub fn main_texture(app: &mut App, entity: Entity) -> Result { + upload_dirty(app, entity)?; + Ok(view_target(app, entity)?.main_texture().clone()) +} + +/// Drops the CPU copy after something other than a flush wrote the canvas texture +pub fn invalidate_pixels(app: &mut App, entity: Entity) { + if let Some(mut graphics) = app.world_mut().get_mut::(entity) { + graphics.pixels = None; + graphics.dirty = None; + } +} + +/// Makes the CPU copy current: one readback serves every read until the next draw +fn ensure_pixels(app: &mut App, entity: Entity) -> Result<()> { + let cached = app + .world() + .get::(entity) + .ok_or(ProcessingError::GraphicsNotFound)? + .pixels + .is_some(); + if cached && !has_commands(app, entity) { + return Ok(()); + } + flush(app, entity)?; + let texture = view_target(app, entity)?.main_texture().clone(); + app.world_mut() + .run_system_cached_with(readback_raw, (entity, texture)) + .unwrap() +} + +/// The canvas pixels in its texture format +pub fn read_pixels(app: &mut App, entity: Entity) -> Result { + ensure_pixels(app, entity)?; + let graphics = app + .world() + .get::(entity) + .ok_or(ProcessingError::GraphicsNotFound)?; + Ok(ReadbackData { + bytes: graphics.pixels.clone().unwrap_or_default(), + format: graphics.texture_format, + width: graphics.size.width, + height: graphics.size.height, + }) +} + +/// Physical pixels per logical pixel on each axis +fn pixel_scale(app: &App, entity: Entity) -> Result<(u32, u32, f64, f64)> { + let world = app.world(); + let size = world + .get::(entity) + .ok_or(ProcessingError::GraphicsNotFound)? + .size; + let (lw, lh) = world + .get::(entity) + .map_or((size.width, size.height), |s| (s.0.max(1), s.1.max(1))); + Ok(( + lw, + lh, + size.width as f64 / lw as f64, + size.height as f64 / lh as f64, + )) +} + +/// The pixel at logical (`x`, `y`); `None` outside the canvas +pub fn get_pixel(app: &mut App, entity: Entity, x: i64, y: i64) -> Result> { + let (lw, lh, sx, sy) = pixel_scale(app, entity)?; + if x < 0 || y < 0 || x >= lw as i64 || y >= lh as i64 { + return Ok(None); + } + ensure_pixels(app, entity)?; + let graphics = app + .world() + .get::(entity) + .ok_or(ProcessingError::GraphicsNotFound)?; + let Some(data) = graphics.pixels.as_deref() else { + return Ok(None); + }; + let px_size = pixel_size(graphics.texture_format)?; + let px = ((x as f64 * sx) as usize).min(graphics.size.width as usize - 1); + let py = ((y as f64 * sy) as usize).min(graphics.size.height as usize - 1); + let at = (py * graphics.size.width as usize + px) * px_size; + crate::image::decode_pixel(&data[at..at + px_size], graphics.texture_format).map(Some) +} + +/// Sets the pixel at logical (`x`, `y`), which covers a block of physical pixels +pub fn set_pixel(app: &mut App, entity: Entity, x: i64, y: i64, color: LinearRgba) -> Result<()> { + let (lw, lh, sx, sy) = pixel_scale(app, entity)?; + if x < 0 || y < 0 || x >= lw as i64 || y >= lh as i64 { + return Ok(()); + } + let x0 = (x as f64 * sx) as i64; + let y0 = (y as f64 * sy) as i64; + let w = (((x + 1) as f64 * sx) as i64 - x0).max(1) as u32; + let h = (((y + 1) as f64 * sy) as i64 - y0).max(1) as u32; + write_region(app, entity, x0, y0, w, h, &vec![color; (w * h) as usize]) +} + +/// Writes a block of physical pixels at (`x`, `y`); parts outside the canvas are dropped +pub fn write_region( + app: &mut App, + entity: Entity, + x: i64, + y: i64, + width: u32, + height: u32, + pixels: &[LinearRgba], +) -> Result<()> { + if pixels.len() != (width * height) as usize { + return Err(ProcessingError::InvalidArgument(format!( + "Expected {} pixels for {width}x{height} region, got {}", + width * height, + pixels.len() + ))); + } + let size = app + .world() + .get::(entity) + .ok_or(ProcessingError::GraphicsNotFound)? + .size; + let x0 = x.max(0); + let y0 = y.max(0); + let x1 = (x + width as i64).min(size.width as i64); + let y1 = (y + height as i64).min(size.height as i64); + if x1 <= x0 || y1 <= y0 { + return Ok(()); + } + // draws queued before this write must land under it + if has_commands(app, entity) { + flush(app, entity)?; + } + let (cw, ch) = ((x1 - x0) as u32, (y1 - y0) as u32); + let clipped: Vec = (y0..y1) + .flat_map(|dy| { + let row = (dy - y) as usize * width as usize + (x0 - x) as usize; + pixels[row..row + cw as usize].iter().copied() + }) + .collect(); + let rect = URect::new(x0 as u32, y0 as u32, x1 as u32, y1 as u32); + + let mut graphics = app + .world_mut() + .get_mut::(entity) + .ok_or(ProcessingError::GraphicsNotFound)?; + let format = graphics.texture_format; + let px_size = pixel_size(format)?; + let bytes = pixels_to_bytes(&clipped, format)?; + let graphics = &mut *graphics; + if let Some(data) = graphics.pixels.as_mut() { + // the CPU copy is current, so patch it and upload the touched rect once at the next flush + let stride = size.width as usize * px_size; + let run = cw as usize * px_size; + for row in 0..ch as usize { + let dst = (y0 as usize + row) * stride + x0 as usize * px_size; + data[dst..dst + run].copy_from_slice(&bytes[row * run..(row + 1) * run]); + } + graphics.dirty = Some(graphics.dirty.map_or(rect, |d| d.union(rect))); + return Ok(()); + } + let texture = view_target(app, entity)?.main_texture().clone(); + app.world_mut() + .run_system_cached_with( + update_region_write, + ( + entity, + texture, + x0 as u32, + y0 as u32, + cw, + ch, + bytes, + px_size as u32, + ), + ) + .unwrap() +} + pub fn apply_filter(app: &mut App, graphics: Entity, filter: Entity) -> Result<()> { filter::apply(app, graphics, filter) } @@ -676,11 +905,21 @@ pub fn end_draw(app: &mut App, entity: Entity) -> Result<()> { /// // TODO: why is metal particularly affected by this? can we remove this? pub fn warmup(app: &mut App, entity: Entity) -> Result<()> { + // an offscreen buffer starts transparent, like Processing's createGraphics + let offscreen = matches!( + app.world().get::(entity), + Some(RenderTarget::Image(_)) + ); + let clear = if offscreen { + Color::NONE + } else { + DEFAULT_CLEAR_COLOR + }; for _ in 0..3 { app.world_mut() .run_system_cached_with( record_command, - (entity, DrawCommand::BackgroundColor(DEFAULT_CLEAR_COLOR)), + (entity, DrawCommand::BackgroundColor(clear)), ) .unwrap()?; present(app, entity)?; @@ -709,12 +948,12 @@ pub struct ReadbackData { pub fn readback_raw( In((entity, texture)): In<(Entity, Texture)>, - graphics_query: Query<&Graphics>, + mut graphics_query: Query<&mut Graphics>, render_device: Res, render_queue: Res, -) -> Result { - let graphics = graphics_query - .get(entity) +) -> Result<()> { + let mut graphics = graphics_query + .get_mut(entity) .map_err(|_| ProcessingError::GraphicsNotFound)?; let mut encoder = render_device.create_command_encoder(&CommandEncoderDescriptor::default()); @@ -773,12 +1012,9 @@ pub fn readback_raw( data }; - Ok(ReadbackData { - bytes: unpadded, - format: graphics.texture_format, - width: graphics.size.width, - height: graphics.size.height, - }) + graphics.pixels = Some(unpadded); + graphics.dirty = None; + Ok(()) } pub fn update_region_write( diff --git a/crates/processing_render/src/image.rs b/crates/processing_render/src/image.rs index 57072cb8..fee055d7 100644 --- a/crates/processing_render/src/image.rs +++ b/crates/processing_render/src/image.rs @@ -3,6 +3,7 @@ //! It can be created from raw pixel data, loaded from disk, resized, and read back to CPU memory. use std::path::PathBuf; +use bevy::color::Srgba; use bevy::{ asset::{ AssetPath, LoadState, RenderAssetUsages, handle_internal_asset_events, @@ -12,7 +13,7 @@ use bevy::{ image::{ImageAddressMode, ImageFilterMode, ImageSampler, ImageSamplerDescriptor}, prelude::*, render::{ - Render, RenderApp, RenderSystems, + ExtractSchedule, MainWorld, Render, RenderApp, RenderSystems, extract_component::{ExtractComponent, ExtractComponentPlugin}, render_asset::RenderAssets, render_resource::{ @@ -38,6 +39,7 @@ impl Plugin for ImagePlugin { if let Some(render_app) = app.get_sub_app_mut(RenderApp) { render_app.init_resource::(); render_app.add_systems(Render, blit_backgrounds.in_set(RenderSystems::Prepare)); + render_app.add_systems(ExtractSchedule, upload_pending); } } } @@ -207,6 +209,67 @@ pub struct Image { readback_buffer: Buffer, pub texture_format: TextureFormat, pub size: Extent3d, + /// Any pixel has alpha below one, so draws sampling it must blend + pub translucent: bool, + pub coherence: Coherence, + /// Bound as a storage texture, so any compute dispatch may write it + pub bound_rw: bool, +} + +/// Which copy of the pixels is newer; the other is brought up to date on demand +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Coherence { + /// CPU writes inside the rect are not in the texture yet + CpuAhead(URect), + InSync, + /// The GPU wrote the texture, so the CPU copy is stale or absent + GpuAhead, +} + +impl Coherence { + fn of_asset(image: &bevy::image::Image) -> Self { + if image.data.is_some() { + Self::InSync + } else { + Self::GpuAhead + } + } +} + +impl Image { + fn cpu_wrote(&mut self, rect: URect) { + self.coherence = match self.coherence { + Coherence::CpuAhead(pending) => Coherence::CpuAhead(pending.union(rect)), + _ => Coherence::CpuAhead(rect), + }; + } +} + +/// Whether tightly packed `data` in `format` holds any pixel with alpha below one +pub fn has_translucent_pixels(data: &[u8], format: TextureFormat, width: u32, height: u32) -> bool { + let Ok(px_size) = pixel_size(format) else { + return true; + }; + if px_size == 4 { + return data.as_chunks::<4>().0.iter().any(|p| p[3] < u8::MAX); + } + match bytes_to_pixels(data, format, width, height, width as usize * px_size) { + Ok(pixels) => pixels.iter().any(|p| p.alpha < 1.0), + Err(_) => true, + } +} + +/// `translucent` for a CPU-side image; unknown contents count as translucent +fn image_translucent(image: &bevy::image::Image) -> bool { + let size = image.texture_descriptor.size; + image.data.as_deref().is_none_or(|data| { + has_translucent_pixels( + data, + image.texture_descriptor.format, + size.width, + size.height, + ) + }) } pub fn create( @@ -215,6 +278,7 @@ pub fn create( mut images: ResMut>, render_device: Res, ) -> Entity { + let translucent = has_translucent_pixels(&data, texture_format, size.width, size.height); let mut image = bevy::image::Image::new( size, TextureDimension::D2, @@ -242,6 +306,9 @@ pub fn create( readback_buffer, texture_format, size, + translucent, + coherence: Coherence::InSync, + bound_rw: false, },)) .id() } @@ -266,6 +333,8 @@ pub fn from_handle( let size = image.texture_descriptor.size; let texture_format = image.texture_descriptor.format; + let translucent = image_translucent(image); + let coherence = Coherence::of_asset(image); let render_device = world.resource::(); let readback_buffer = create_readback_buffer( @@ -282,6 +351,9 @@ pub fn from_handle( readback_buffer, texture_format, size, + translucent, + coherence, + bound_rw: false, }) .id()) } @@ -305,6 +377,8 @@ pub fn load(In(path): In, world: &mut World) -> Result { let size = image.texture_descriptor.size; let texture_format = image.texture_descriptor.format; + let translucent = image_translucent(image); + let coherence = Coherence::of_asset(image); let render_device = world.resource::(); let readback_buffer = create_readback_buffer( @@ -321,6 +395,9 @@ pub fn load(In(path): In, world: &mut World) -> Result { readback_buffer, texture_format, size, + translucent, + coherence, + bound_rw: false, }) .id()) } @@ -339,6 +416,7 @@ pub fn resize( .get_mut(&p_image.handle) .ok_or(ProcessingError::ImageNotFound)? .resize_in_place(new_size); + p_image.coherence = Coherence::InSync; p_image.readback_buffer = create_readback_buffer( &render_device, @@ -354,13 +432,13 @@ pub fn resize( pub fn readback( In((entity, texture)): In<(Entity, Texture)>, - p_images: Query<&Image>, + mut p_images: Query<&mut Image>, mut images: ResMut>, render_device: Res, render_queue: ResMut, -) -> Result> { - let p_image = p_images - .get(entity) +) -> Result<()> { + let mut p_image = p_images + .get_mut(entity) .map_err(|_| ProcessingError::ImageNotFound)?; let mut encoder = render_device.create_command_encoder(&CommandEncoderDescriptor::default()); @@ -403,99 +481,361 @@ pub fn readback( r.recv().expect("Failed to receive the map_async message"); - let data = buffer_slice.get_mapped_range().to_vec(); - - let mut image = images - .get_mut(&p_image.handle) - .ok_or(ProcessingError::ImageNotFound)?; - image.data = Some(data.clone()); - + // drop the row padding so the stored copy matches the asset's tight layout + let bytes_per_row = p_image.size.width as usize * px_size; + let data: Vec = buffer_slice + .get_mapped_range() + .chunks_exact(padded_bytes_per_row) + .take(p_image.size.height as usize) + .flat_map(|row| row[..bytes_per_row].iter().copied()) + .collect(); p_image.readback_buffer.unmap(); + // untracked: the GPU already holds these pixels, so no re-upload is wanted + images + .get_mut_untracked(&p_image.handle) + .ok_or(ProcessingError::ImageNotFound)? + .data = Some(data); + p_image.coherence = Coherence::InSync; + Ok(()) +} + +/// Pixels from the asset's CPU copy; `sync_cpu` first if the GPU may be ahead +pub fn cpu_pixels(world: &World, entity: Entity) -> Result> { + let p_image = world + .get::(entity) + .ok_or(ProcessingError::ImageNotFound)?; + let image = world + .resource::>() + .get(&p_image.handle) + .ok_or(ProcessingError::ImageNotFound)?; + let data = image + .data + .as_deref() + .ok_or(ProcessingError::ImageNotFound)?; + let px_size = pixel_size(p_image.texture_format)?; + let stride = p_image.size.width as usize * px_size; + check_len(data.len(), stride, p_image.size, p_image.texture_format)?; bytes_to_pixels( - &data, + data, p_image.texture_format, p_image.size.width, p_image.size.height, - padded_bytes_per_row, + stride, ) } -pub fn update_region_write( - In((entity, texture, x, y, width, height, data, px_size)): In<( - Entity, - Texture, - u32, - u32, - u32, - u32, - Vec, - u32, - )>, - p_images: Query<&Image>, - render_queue: Res, -) -> Result<()> { - let p_image = p_images - .get(entity) - .map_err(|_| ProcessingError::ImageNotFound)?; +/// A CPU buffer that disagrees with the image size would make row math index out of bounds +fn check_len(len: usize, stride: usize, size: Extent3d, format: TextureFormat) -> Result<()> { + let expected = stride * size.height as usize; + if len != expected { + return Err(ProcessingError::InvalidArgument(format!( + "image data is {len} bytes, expected {expected} for {}x{} {format:?}", + size.width, size.height + ))); + } + Ok(()) +} + +/// One pixel from the asset's CPU copy; `None` outside the image +pub fn cpu_pixel(world: &World, entity: Entity, x: i64, y: i64) -> Result> { + let p_image = world + .get::(entity) + .ok_or(ProcessingError::ImageNotFound)?; + let size = p_image.size; + if x < 0 || y < 0 || x >= size.width as i64 || y >= size.height as i64 { + return Ok(None); + } + let data = world + .resource::>() + .get(&p_image.handle) + .and_then(|image| image.data.as_deref()) + .ok_or(ProcessingError::ImageNotFound)?; + let px_size = pixel_size(p_image.texture_format)?; + let stride = size.width as usize * px_size; + check_len(data.len(), stride, size, p_image.texture_format)?; + let at = y as usize * stride + x as usize * px_size; + decode_pixel(&data[at..at + px_size], p_image.texture_format).map(Some) +} + +/// Replaces the asset's CPU pixels with `data` in the texture format +pub fn cpu_write(world: &mut World, entity: Entity, data: Vec) -> Result<()> { + let (handle, texture_format, size) = { + let p_image = world + .get::(entity) + .ok_or(ProcessingError::ImageNotFound)?; + (p_image.handle.clone(), p_image.texture_format, p_image.size) + }; + let stride = size.width as usize * pixel_size(texture_format)?; + check_len(data.len(), stride, size, texture_format)?; + // untracked: only the dirty rect is uploaded, not the whole asset + world + .resource_mut::>() + .get_mut_untracked(&handle) + .ok_or(ProcessingError::ImageNotFound)? + .data = Some(data); + let full = URect::new(0, 0, size.width, size.height); + world + .get_mut::(entity) + .ok_or(ProcessingError::ImageNotFound)? + .coherence = Coherence::CpuAhead(full); + Ok(()) +} - if x + width > p_image.size.width || y + height > p_image.size.height { +/// Writes a `width`×`height` block of `pixels` at (`x`, `y`) into the asset's CPU pixels, +/// clipping to the image like Processing's `set`; `sync_cpu` first if the GPU may be ahead +pub fn cpu_write_region( + world: &mut World, + entity: Entity, + x: i64, + y: i64, + width: u32, + height: u32, + pixels: &[LinearRgba], +) -> Result<()> { + let expected_count = (width * height) as usize; + if pixels.len() != expected_count { return Err(ProcessingError::InvalidArgument(format!( - "Region ({}, {}, {}, {}) exceeds image bounds ({}, {})", - x, y, width, height, p_image.size.width, p_image.size.height + "Expected {} pixels for {}x{} region, got {}", + expected_count, + width, + height, + pixels.len() ))); } + let (handle, texture_format, size) = { + let p_image = world + .get::(entity) + .ok_or(ProcessingError::ImageNotFound)?; + (p_image.handle.clone(), p_image.texture_format, p_image.size) + }; + let x0 = x.max(0); + let y0 = y.max(0); + let x1 = (x + width as i64).min(size.width as i64); + let y1 = (y + height as i64).min(size.height as i64); + if x1 <= x0 || y1 <= y0 { + return Ok(()); + } + let px_size = pixel_size(texture_format)?; + let bytes = pixels_to_bytes(pixels, texture_format)?; + let stride = size.width as usize * px_size; + let mut images = world.resource_mut::>(); + // untracked: only the dirty rect is uploaded, not the whole asset + let data = images + .get_mut_untracked(&handle) + .and_then(|image| image.data.as_mut()) + .ok_or(ProcessingError::ImageNotFound)?; + check_len(data.len(), stride, size, texture_format)?; + let run = (x1 - x0) as usize * px_size; + for dy in y0..y1 { + let src = ((dy - y) as usize * width as usize + (x0 - x) as usize) * px_size; + let dst = dy as usize * stride + x0 as usize * px_size; + data[dst..dst + run].copy_from_slice(&bytes[src..src + run]); + } + world + .get_mut::(entity) + .ok_or(ProcessingError::ImageNotFound)? + .cpu_wrote(URect::new(x0 as u32, y0 as u32, x1 as u32, y1 as u32)); + Ok(()) +} - let bytes_per_row = width * px_size; +/// Writes `rect` of the asset's CPU pixels into the texture +fn upload_rect( + queue: &RenderQueue, + texture: &Texture, + image: &bevy::image::Image, + rect: URect, +) -> Result<()> { + let Some(data) = image.data.as_deref() else { + return Ok(()); + }; + let size = image.texture_descriptor.size; + let format = image.texture_descriptor.format; + write_rect(queue, texture, data, size, format, rect) +} - render_queue.write_texture( +/// Writes `rect` of tightly packed `data`, which covers a texture of `size`, into the texture +pub(crate) fn write_rect( + queue: &RenderQueue, + texture: &Texture, + data: &[u8], + size: Extent3d, + format: TextureFormat, + rect: URect, +) -> Result<()> { + let px_size = pixel_size(format)?; + let stride = size.width as usize * px_size; + check_len(data.len(), stride, size, format)?; + let start = rect.min.y as usize * stride + rect.min.x as usize * px_size; + let len = (rect.height() as usize - 1) * stride + rect.width() as usize * px_size; + queue.write_texture( TexelCopyTextureInfo { - texture: &texture, + texture, mip_level: 0, - origin: Origin3d { x, y, z: 0 }, + origin: Origin3d { + x: rect.min.x, + y: rect.min.y, + z: 0, + }, aspect: Default::default(), }, - &data, + &data[start..start + len], TexelCopyBufferLayout { offset: 0, - bytes_per_row: Some(bytes_per_row), + bytes_per_row: Some(stride as u32), rows_per_image: None, }, Extent3d { - width, - height, + width: rect.width(), + height: rect.height(), depth_or_array_layers: 1, }, ); + Ok(()) +} + +/// Uploads every pending CPU write once per frame, before anything renders +fn upload_pending( + mut main_world: ResMut, + mut state: Local>>, + queue: Res, + gpu_images: Res>, +) { + let state = state.get_or_insert_with(|| main_world.query()); + let pending: Vec<_> = state + .iter(&main_world) + .filter_map(|(entity, p_image)| match p_image.coherence { + Coherence::CpuAhead(rect) => Some((entity, p_image.handle.clone(), rect)), + _ => None, + }) + .collect(); + for (entity, handle, rect) in pending { + // an unprepared image uploads its whole CPU copy when Bevy first prepares it + let Some(gpu_image) = gpu_images.get(&handle) else { + continue; + }; + let Some(image) = main_world + .resource::>() + .get(&handle) + else { + continue; + }; + if let Err(err) = upload_rect(&queue, &gpu_image.texture, image, rect) { + error!("image upload failed: {err}"); + } + if let Some(mut p_image) = main_world.get_mut::(entity) { + p_image.coherence = Coherence::InSync; + } + } +} +/// Uploads an image's pending CPU writes now, for GPU work that runs outside a frame +pub fn sync_gpu(app: &mut App, entity: Entity) -> Result<()> { + let p_image = app + .world() + .get::(entity) + .ok_or(ProcessingError::ImageNotFound)?; + let Coherence::CpuAhead(rect) = p_image.coherence else { + return Ok(()); + }; + let handle = p_image.handle.clone(); + let gpu_images = app + .sub_app(RenderApp) + .world() + .resource::>(); + // an unprepared image uploads its whole CPU copy when Bevy first prepares it + let Some(gpu_image) = gpu_images.get(&handle) else { + return Ok(()); + }; + let world = app.world(); + let image = world + .resource::>() + .get(&handle) + .ok_or(ProcessingError::ImageNotFound)?; + upload_rect( + world.resource::(), + &gpu_image.texture, + image, + rect, + )?; + app.world_mut() + .get_mut::(entity) + .ok_or(ProcessingError::ImageNotFound)? + .coherence = Coherence::InSync; Ok(()) } -pub fn prepare_update_region( - world: &World, - entity: Entity, - width: u32, - height: u32, - pixels: &[LinearRgba], -) -> Result<(Vec, u32)> { - let expected_count = (width * height) as usize; - if pixels.len() != expected_count { - return Err(ProcessingError::InvalidArgument(format!( - "Expected {} pixels for {}x{} region, got {}", - expected_count, - width, - height, - pixels.len() - ))); +/// `sync_gpu` for every image +pub fn flush_uploads(app: &mut App) -> Result<()> { + let world = app.world_mut(); + let pending: Vec = world + .query::<(Entity, &Image)>() + .iter(world) + .filter(|(_, p_image)| matches!(p_image.coherence, Coherence::CpuAhead(_))) + .map(|(entity, _)| entity) + .collect(); + for entity in pending { + sync_gpu(app, entity)?; } + Ok(()) +} +/// Reads GPU-written pixels back into the CPU copy, so CPU reads and writes see them +pub fn sync_cpu(app: &mut App, entity: Entity) -> Result<()> { + let world = app.world(); let p_image = world .get::(entity) .ok_or(ProcessingError::ImageNotFound)?; - let px_size = pixel_size(p_image.texture_format)? as u32; - let data = pixels_to_bytes(pixels, p_image.texture_format)?; + let has_cpu = world + .resource::>() + .get(&p_image.handle) + .is_some_and(|image| image.data.is_some()); + if has_cpu && p_image.coherence != Coherence::GpuAhead { + return Ok(()); + } + let texture = gpu_texture(app, entity)?; + app.world_mut() + .run_system_cached_with(readback, (entity, texture)) + .unwrap() +} + +/// After a compute dispatch, every storage-bound image may hold new pixels +pub fn invalidate_rw(mut p_images: Query<&mut Image>) { + for mut p_image in &mut p_images { + if p_image.bound_rw && p_image.coherence != Coherence::GpuAhead { + p_image.coherence = Coherence::GpuAhead; + } + } +} - Ok((data, px_size)) +/// Marks an image whose texture the GPU just wrote +pub fn gpu_wrote(app: &mut App, entity: Entity) -> Result<()> { + app.world_mut() + .get_mut::(entity) + .ok_or(ProcessingError::ImageNotFound)? + .coherence = Coherence::GpuAhead; + Ok(()) +} + +/// The texture for a GPU reader, with every CPU write uploaded +pub fn texture_for_read(app: &mut App, entity: Entity) -> Result { + sync_gpu(app, entity)?; + gpu_texture(app, entity) +} + +/// The texture for a GPU writer; the CPU copy is stale from here on +pub fn texture_for_write(app: &mut App, entity: Entity) -> Result { + let texture = texture_for_read(app, entity)?; + gpu_wrote(app, entity)?; + Ok(texture) +} + +fn gpu_texture(app: &mut App, entity: Entity) -> Result { + // a new image has no texture until a frame prepares it + if gpu_image(app, entity).is_err() { + app.update(); + } + Ok(gpu_image(app, entity)?.texture.clone()) } /// Get the pixel dimensions `(width, height)` of an image. @@ -532,9 +872,11 @@ pub fn pixel_size(format: TextureFormat) -> Result { /// Convert LinearRgba pixels to raw bytes in the specified texture format. pub fn pixels_to_bytes(pixels: &[LinearRgba], format: TextureFormat) -> Result> { match format { - TextureFormat::Rgba8Unorm | TextureFormat::Rgba8UnormSrgb => { - Ok(pixels.iter().flat_map(|p| p.to_u8_array()).collect()) - } + TextureFormat::Rgba8Unorm => Ok(pixels.iter().flat_map(|p| p.to_u8_array()).collect()), + TextureFormat::Rgba8UnormSrgb => Ok(pixels + .iter() + .flat_map(|p| Srgba::from(*p).to_u8_array()) + .collect()), TextureFormat::Rgba16Float => Ok(pixels .iter() .flat_map(|p| { @@ -578,48 +920,27 @@ pub fn bytes_to_pixels( ) -> Result> { let px_size = pixel_size(format)?; let bytes_per_row = width as usize * px_size; + data.chunks_exact(padded_bytes_per_row) + .take(height as usize) + .flat_map(|row| row[..bytes_per_row].chunks_exact(px_size)) + .map(|chunk| decode_pixel(chunk, format)) + .collect() +} - let pixels = match format { - TextureFormat::Rgba8Unorm | TextureFormat::Rgba8UnormSrgb => data - .chunks_exact(padded_bytes_per_row) - .take(height as usize) - .flat_map(|row| { - row[..bytes_per_row].chunks_exact(px_size).map(|chunk| { - LinearRgba::from_u8_array([chunk[0], chunk[1], chunk[2], chunk[3]]) - }) - }) - .collect(), - TextureFormat::Rgba16Float => data - .chunks_exact(padded_bytes_per_row) - .take(height as usize) - .flat_map(|row| { - row[..bytes_per_row].chunks_exact(px_size).map(|chunk| { - let r = f16::from_bits(u16::from_le_bytes([chunk[0], chunk[1]])).to_f32(); - let g = f16::from_bits(u16::from_le_bytes([chunk[2], chunk[3]])).to_f32(); - let b = f16::from_bits(u16::from_le_bytes([chunk[4], chunk[5]])).to_f32(); - let a = f16::from_bits(u16::from_le_bytes([chunk[6], chunk[7]])).to_f32(); - LinearRgba::from_f32_array([r, g, b, a]) - }) - }) - .collect(), - TextureFormat::Rgba32Float => data - .chunks_exact(padded_bytes_per_row) - .take(height as usize) - .flat_map(|row| { - row[..bytes_per_row].chunks_exact(px_size).map(|chunk| { - let r = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); - let g = f32::from_le_bytes([chunk[4], chunk[5], chunk[6], chunk[7]]); - let b = f32::from_le_bytes([chunk[8], chunk[9], chunk[10], chunk[11]]); - let a = f32::from_le_bytes([chunk[12], chunk[13], chunk[14], chunk[15]]); - LinearRgba::from_f32_array([r, g, b, a]) - }) - }) - .collect(), - // TODO: Handle more formats as needed +pub(crate) fn decode_pixel(c: &[u8], format: TextureFormat) -> Result { + Ok(match format { + TextureFormat::Rgba8Unorm => LinearRgba::from_u8_array([c[0], c[1], c[2], c[3]]), + TextureFormat::Rgba8UnormSrgb => Srgba::from_u8_array([c[0], c[1], c[2], c[3]]).into(), + TextureFormat::Rgba16Float => { + let ch = |i: usize| f16::from_bits(u16::from_le_bytes([c[i], c[i + 1]])).to_f32(); + LinearRgba::from_f32_array([ch(0), ch(2), ch(4), ch(6)]) + } + TextureFormat::Rgba32Float => { + let ch = |i: usize| f32::from_le_bytes([c[i], c[i + 1], c[i + 2], c[i + 3]]); + LinearRgba::from_f32_array([ch(0), ch(4), ch(8), ch(12)]) + } _ => return Err(ProcessingError::UnsupportedTextureFormat), - }; - - Ok(pixels) + }) } /// Create a readback buffer for the given texture dimensions and format. @@ -686,7 +1007,7 @@ pub fn set_sampler( Ok(()) } -pub fn gpu_image(app: &mut App, entity: Entity) -> Result<&GpuImage> { +fn gpu_image(app: &mut App, entity: Entity) -> Result<&GpuImage> { let handle = app .world() .get::(entity) diff --git a/crates/processing_render/src/lib.rs b/crates/processing_render/src/lib.rs index f79e86a8..c90ac0c8 100644 --- a/crates/processing_render/src/lib.rs +++ b/crates/processing_render/src/lib.rs @@ -65,7 +65,6 @@ use processing_core::error; use crate::geometry::{AttributeFormat, AttributeValue}; use crate::graphics::flush; -use crate::image::gpu_image; use crate::render::command::DrawCommand; #[derive(Component)] @@ -665,14 +664,7 @@ pub fn graphics_destroy(graphics_entity: Entity) -> error::Result<()> { /// Read back raw pixel data from the graphics surface. pub fn graphics_readback_raw(graphics_entity: Entity) -> error::Result { - app_mut(|app| { - graphics::flush(app, graphics_entity)?; - let vt = graphics::view_target(app, graphics_entity)?; - let texture = vt.main_texture().clone(); - app.world_mut() - .run_system_cached_with(graphics::readback_raw, (graphics_entity, texture)) - .unwrap() - }) + app_mut(|app| graphics::read_pixels(app, graphics_entity)) } /// Read back pixel data from the graphics surface as LinearRgba. @@ -692,60 +684,46 @@ pub fn graphics_readback(graphics_entity: Entity) -> error::Result error::Result<()> { app_mut(|app| { - let vt = graphics::view_target(app, graphics_entity)?; - let texture = vt.main_texture().clone(); - let world = app.world_mut(); - let size = world + let size = app + .world() .get::(graphics_entity) .ok_or(error::ProcessingError::GraphicsNotFound)? .size; - let (data, px_size) = graphics::prepare_update_region( - world, - graphics_entity, - size.width, - size.height, - pixels, - )?; - world - .run_system_cached_with( - graphics::update_region_write, - ( - graphics_entity, - texture, - 0, - 0, - size.width, - size.height, - data, - px_size, - ), - ) - .unwrap() + graphics::write_region(app, graphics_entity, 0, 0, size.width, size.height, pixels) }) } /// Update a region of the graphics surface with new pixel data. +/// The pixel at logical (`x`, `y`); `None` outside the surface, where Processing's `get` is +/// transparent black +pub fn graphics_get_pixel( + graphics_entity: Entity, + x: i64, + y: i64, +) -> error::Result> { + app_mut(|app| graphics::get_pixel(app, graphics_entity, x, y)) +} + +/// Set the pixel at logical (`x`, `y`); outside the surface it is dropped. +pub fn graphics_set_pixel( + graphics_entity: Entity, + x: i64, + y: i64, + color: LinearRgba, +) -> error::Result<()> { + app_mut(|app| graphics::set_pixel(app, graphics_entity, x, y, color)) +} + +/// Write a block of physical pixels at (`x`, `y`); parts outside the surface are dropped. pub fn graphics_update_region( graphics_entity: Entity, - x: u32, - y: u32, + x: i64, + y: i64, width: u32, height: u32, pixels: &[LinearRgba], ) -> error::Result<()> { - app_mut(|app| { - let vt = graphics::view_target(app, graphics_entity)?; - let texture = vt.main_texture().clone(); - let world = app.world_mut(); - let (data, px_size) = - graphics::prepare_update_region(world, graphics_entity, width, height, pixels)?; - world - .run_system_cached_with( - graphics::update_region_write, - (graphics_entity, texture, x, y, width, height, data, px_size), - ) - .unwrap() - }) + app_mut(|app| graphics::write_region(app, graphics_entity, x, y, width, height, pixels)) } /// Set the color mode for a graphics entity. @@ -1189,6 +1167,7 @@ pub async fn image_load(path: &str) -> error::Result { /// Resize an existing image to new size. pub fn image_resize(entity: Entity, new_size: Extent3d) -> error::Result<()> { app_mut(|app| { + image::sync_cpu(app, entity)?; app.world_mut() .run_system_cached_with(image::resize, (entity, new_size)) .unwrap() @@ -1224,55 +1203,58 @@ pub fn image_copy_from(dst: Entity, src: Entity, src_is_graphics: bool) -> error .clone(); image::BlitSource::Image(handle) }; - let dst_handle = app - .world() - .get::(dst) - .ok_or(error::ProcessingError::ImageNotFound)? - .handle - .clone(); + // a render target's alpha is unknown until read back, so it is assumed translucent + let translucent = match &source { + image::BlitSource::Graphics(_) => true, + image::BlitSource::Image(_) => app + .world() + .get::(src) + .is_none_or(|img| img.translucent), + }; + let mut dst_image = app + .world_mut() + .get_mut::(dst) + .ok_or(error::ProcessingError::ImageNotFound)?; + dst_image.translucent = translucent; + let dst_handle = dst_image.handle.clone(); app.sub_app_mut(bevy::render::RenderApp) .world_mut() .run_system_cached_with(image::blit, (dst_handle, source)) - .unwrap() + .unwrap()?; + image::gpu_wrote(app, dst) + }) +} + +/// One pixel of an image; `None` outside it, where Processing's `get` is transparent black +pub fn image_get_pixel(entity: Entity, x: i64, y: i64) -> error::Result> { + app_mut(|app| { + image::sync_cpu(app, entity)?; + image::cpu_pixel(app.world(), entity, x, y) }) } -/// Read back image data from GPU to CPU. +/// The image's pixels; served from the CPU copy unless the GPU wrote the texture pub fn image_readback(entity: Entity) -> error::Result> { app_mut(|app| { - let texture = gpu_image(app, entity)?.texture.clone(); - app.world_mut() - .run_system_cached_with(image::readback, (entity, texture)) - .unwrap() + image::sync_cpu(app, entity)?; + image::cpu_pixels(app.world(), entity) }) } -/// Update an existing image with new pixel data. +/// Replace every pixel of an image. pub fn image_update(entity: Entity, pixels: &[LinearRgba]) -> error::Result<()> { app_mut(|app| { - let texture = gpu_image(app, entity)?.texture.clone(); let world = app.world_mut(); let size = world .get::(entity) .ok_or(error::ProcessingError::ImageNotFound)? .size; - let (data, px_size) = - image::prepare_update_region(world, entity, size.width, size.height, pixels)?; + image::cpu_write_region(world, entity, 0, 0, size.width, size.height, pixels)?; world - .run_system_cached_with( - image::update_region_write, - ( - entity, - texture, - 0, - 0, - size.width, - size.height, - data, - px_size, - ), - ) - .unwrap() + .get_mut::(entity) + .ok_or(error::ProcessingError::ImageNotFound)? + .translucent = pixels.iter().any(|p| p.alpha < 1.0); + Ok(()) }) } @@ -1280,65 +1262,38 @@ pub fn image_update(entity: Entity, pixels: &[LinearRgba]) -> error::Result<()> /// format. pub fn image_update_raw(entity: Entity, data: &[u8]) -> error::Result<()> { app_mut(|app| { - if gpu_image(app, entity).is_err() && app.world().get::(entity).is_some() { - app.update(); - } - let texture = gpu_image(app, entity)?.texture.clone(); let world = app.world_mut(); - let (size, texture_format) = { - let p_image = world - .get::(entity) - .ok_or(error::ProcessingError::ImageNotFound)?; - (p_image.size, p_image.texture_format) - }; - let px_size = image::pixel_size(texture_format)? as u32; - let expected = (size.width * size.height * px_size) as usize; - if data.len() != expected { - return Err(error::ProcessingError::InvalidArgument(format!( - "expected {expected} bytes for a {}x{} {:?} image, got {}", - size.width, - size.height, - texture_format, - data.len() - ))); - } - world - .run_system_cached_with( - image::update_region_write, - ( - entity, - texture, - 0, - 0, - size.width, - size.height, - data.to_vec(), - px_size, - ), - ) - .unwrap() + image::cpu_write(world, entity, data.to_vec())?; + let mut p_image = world + .get_mut::(entity) + .ok_or(error::ProcessingError::ImageNotFound)?; + let (size, texture_format) = (p_image.size, p_image.texture_format); + p_image.translucent = + image::has_translucent_pixels(data, texture_format, size.width, size.height); + Ok(()) }) } -/// Update a region of an existing image with new pixel data. +/// Write a block of pixels at (`x`, `y`); parts outside the image are dropped. pub fn image_update_region( entity: Entity, - x: u32, - y: u32, + x: i64, + y: i64, width: u32, height: u32, pixels: &[LinearRgba], ) -> error::Result<()> { app_mut(|app| { - let texture = gpu_image(app, entity)?.texture.clone(); + image::sync_cpu(app, entity)?; let world = app.world_mut(); - let (data, px_size) = image::prepare_update_region(world, entity, width, height, pixels)?; - world - .run_system_cached_with( - image::update_region_write, - (entity, texture, x, y, width, height, data, px_size), - ) - .unwrap() + image::cpu_write_region(world, entity, x, y, width, height, pixels)?; + if pixels.iter().any(|p| p.alpha < 1.0) { + world + .get_mut::(entity) + .ok_or(error::ProcessingError::ImageNotFound)? + .translucent = true; + } + Ok(()) }) } @@ -2584,6 +2539,8 @@ pub(crate) fn compute_dispatch_no_update( } fn dispatch_inner(app: &mut App, entity: Entity, x: u32, y: u32, z: u32) -> error::Result<()> { + // the dispatch submits outside a frame, so pending image writes go up first + image::flush_uploads(app)?; let args = { let world = app.world(); let c = world @@ -2608,6 +2565,9 @@ fn dispatch_inner(app: &mut App, entity: Entity, x: u32, y: u32, z: u32) -> erro app.world_mut() .run_system_cached(compute::invalidate_rw_buffers) .unwrap(); + app.world_mut() + .run_system_cached(image::invalidate_rw) + .unwrap(); Ok(()) } diff --git a/crates/processing_render/src/render/filter.rs b/crates/processing_render/src/render/filter.rs index 52a2a9b4..00a2cc7a 100644 --- a/crates/processing_render/src/render/filter.rs +++ b/crates/processing_render/src/render/filter.rs @@ -160,6 +160,7 @@ pub fn apply(app: &mut App, graphics: Entity, filter: Entity) -> Result<()> { // flush() also extracts the shader asset to the render world. crate::graphics::flush(app, graphics)?; + crate::graphics::invalidate_pixels(app, graphics); // The pipeline must target the view target's actual texture format, which // bevy may transform from the requested format (HDR, sRGB compositing). diff --git a/crates/processing_render/src/render/mod.rs b/crates/processing_render/src/render/mod.rs index 16717464..b48ff15e 100644 --- a/crates/processing_render/src/render/mod.rs +++ b/crates/processing_render/src/render/mod.rs @@ -252,6 +252,7 @@ pub fn flush_draw_commands( if let Ok(img) = p_images.get(entity) { let mut pbr = state.style.material_key.as_pbr(); pbr.base_color_texture = Some(img.handle.clone()); + pbr.transparent = img.translucent; state.style.material_key = pbr.into(); } } @@ -259,6 +260,7 @@ pub fn flush_draw_commands( if let MaterialKey::Pbr { .. } = &state.style.material_key { let mut pbr = state.style.material_key.as_pbr(); pbr.base_color_texture = None; + pbr.transparent = false; state.style.material_key = pbr.into(); } } @@ -840,7 +842,7 @@ pub fn flush_draw_commands( let tint = state.style.tint_color.unwrap_or(Color::WHITE); let material_key = MaterialKey::Color { - transparent: tint.alpha() < 1.0, + transparent: tint.alpha() < 1.0 || p_image.translucent, background_image: Some(p_image.handle.clone()), uv_transform: uv_xform, blend_state: state.style.blend_state, diff --git a/crates/processing_render/src/shader_property.rs b/crates/processing_render/src/shader_property.rs index a48d1f51..f42e5a0c 100644 --- a/crates/processing_render/src/shader_property.rs +++ b/crates/processing_render/src/shader_property.rs @@ -54,7 +54,7 @@ pub(crate) fn apply_shader_value( name: &str, value: ShaderValue, p_buffers: &mut Query<&mut Buffer>, - p_images: &Query<&PImage>, + p_images: &mut Query<&mut PImage>, ) -> Result<()> { match value { ShaderValue::Buffer(buf_entity) => { @@ -93,10 +93,14 @@ pub(crate) fn apply_shader_value( "property `{name}` expects {category:?}, got Texture", ))); } - let image = p_images - .get(img_entity) + let mut image = p_images + .get_mut(img_entity) .map_err(|_| ProcessingError::ImageNotFound)?; shader.insert(name, image.handle.clone()); + // reflection does not carry the access mode, so any storage binding may write + if category == ParameterCategory::StorageTexture { + image.bound_rw = true; + } Ok(()) } v => { @@ -114,7 +118,7 @@ fn bind_struct( buffers: &[(&str, Entity)], values: &[(&str, ShaderValue)], p_buffers: &mut Query<&mut Buffer>, - p_images: &Query<&PImage>, + p_images: &mut Query<&mut PImage>, ) -> Result<()> { for (suffix, buffer) in buffers { let binding = format!("{name}_{suffix}"); @@ -149,7 +153,7 @@ pub fn set_property( mut computes: Query<&mut Compute>, mut filters: Query<&mut Filter>, mut p_buffers: Query<&mut Buffer>, - p_images: Query<&PImage>, + mut p_images: Query<&mut PImage>, grids: Query<&Grid>, ) -> Result { let shader = if let Ok(compute) = computes.get_mut(entity) { @@ -177,10 +181,10 @@ pub fn set_property( ("dims", ShaderValue::UInt3(grid.params.dims)), ], &mut p_buffers, - &p_images, + &mut p_images, )?; } - other => apply_shader_value(shader, &name, other, &mut p_buffers, &p_images)?, + other => apply_shader_value(shader, &name, other, &mut p_buffers, &mut p_images)?, } Ok(true) } diff --git a/crates/processing_render/src/surface.rs b/crates/processing_render/src/surface.rs index 4feb630f..0660b321 100644 --- a/crates/processing_render/src/surface.rs +++ b/crates/processing_render/src/surface.rs @@ -28,7 +28,7 @@ use bevy::{ render::render_resource::{Extent3d, TextureFormat}, window::{ CompositeAlphaMode, Monitor, RawHandleWrapper, WindowLevel, WindowMode, WindowPosition, - WindowRef, WindowResolution, WindowWrapper, + WindowRef, WindowResized, WindowResolution, WindowScaleFactorChanged, WindowWrapper, }, }; use raw_window_handle::{ @@ -445,6 +445,8 @@ pub fn resize( pub fn set_pixel_density( In((window_entity, density)): In<(Entity, f32)>, mut windows: Query<&mut Window>, + mut resized: bevy::ecs::message::MessageWriter, + mut scale_changed: bevy::ecs::message::MessageWriter, ) -> Result<()> { if let Ok(mut window) = windows.get_mut(window_entity) { let logical_w = window.resolution.width(); @@ -453,6 +455,16 @@ pub fn set_pixel_density( window .resolution .set_physical_resolution((logical_w * density) as u32, (logical_h * density) as u32); + // cameras size their targets from these messages, not from the Window change + scale_changed.write(WindowScaleFactorChanged { + window: window_entity, + scale_factor: density as f64, + }); + resized.write(WindowResized { + window: window_entity, + width: logical_w, + height: logical_h, + }); Ok(()) } else { Err(error::ProcessingError::SurfaceNotFound)