fix(zerocopy): unsafe fn bodies join the unsafe-proof program
#![deny(clippy::undocumented_unsafe_blocks)] never inspected the body of an unsafe fn — operations there are not "unsafe blocks" — so roughly 20 functions' worth of raw GL/CUDA/Vulkan driver calls sat outside the invariant the crate advertises. Concretely, that blind spot is why the constructor-leak and teardown-ordering shapes fixed earlier in this series could ship without ever prompting a reviewer. #![deny(unsafe_op_in_unsafe_fn)] now closes the gap: every unsafe fn body is an explicit unsafe block carrying a SAFETY comment that names the caller contract it relies on (the dlopen'd CUDA wrapper table gets a uniform forward-to-live-table proof). Mechanical; no behavior change. (V2 from design/pf-zerocopy-sweep-handoff.md.) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,7 +11,7 @@
|
|||||||
//! fd we can `poll()` — readable once the producer's writes complete. This makes zero-copy capture
|
//! fd we can `poll()` — readable once the producer's writes complete. This makes zero-copy capture
|
||||||
//! race-free WITHOUT the producer doing anything, *iff* the driver actually attaches the fence. If it
|
//! race-free WITHOUT the producer doing anything, *iff* the driver actually attaches the fence. If it
|
||||||
//! attaches none, the export yields an already-signaled sync_file (poll returns immediately) — no
|
//! attaches none, the export yields an already-signaled sync_file (poll returns immediately) — no
|
||||||
//! wait, no harm, and [`WaitOutcome::NoFence`] tells us the driver doesn't fence (so zero-copy
|
//! wait, no harm, and `WaitOutcome::NoFence` tells us the driver doesn't fence (so zero-copy
|
||||||
//! would still race).
|
//! would still race).
|
||||||
|
|
||||||
// Every `unsafe` block in this file carries a `// SAFETY:` proof; enforce it (unsafe-proof program).
|
// Every `unsafe` block in this file carries a `// SAFETY:` proof; enforce it (unsafe-proof program).
|
||||||
|
|||||||
@@ -249,17 +249,25 @@ fn copy_stream() -> CUstream {
|
|||||||
/// (the source dmabuf is safe to recycle once this returns), but the wait is scoped to our own
|
/// (the source dmabuf is safe to recycle once this returns), but the wait is scoped to our own
|
||||||
/// stream and the copy carries the high priority hint.
|
/// stream and the copy carries the high priority hint.
|
||||||
unsafe fn copy_blocking(copy: &CUDA_MEMCPY2D, what: &str) -> Result<()> {
|
unsafe fn copy_blocking(copy: &CUDA_MEMCPY2D, what: &str) -> Result<()> {
|
||||||
|
// SAFETY: caller contract: the shared context is current and `copy` describes live, in-bounds
|
||||||
|
// device/host memory (each caller carries that proof). Wrapper -> live table; `©` outlives
|
||||||
|
// the synchronous call.
|
||||||
|
unsafe {
|
||||||
let stream = copy_stream();
|
let stream = copy_stream();
|
||||||
ck(cuMemcpy2DAsync_v2(copy, stream), what)?;
|
ck(cuMemcpy2DAsync_v2(copy, stream), what)?;
|
||||||
ck(cuStreamSynchronize(stream), "cuStreamSynchronize")
|
ck(cuStreamSynchronize(stream), "cuStreamSynchronize")
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Issue `copy` on this thread's priority stream WITHOUT waiting — for stream-ordered consumers
|
/// Issue `copy` on this thread's priority stream WITHOUT waiting — for stream-ordered consumers
|
||||||
/// only (the direct-NVENC submit path with `NvEncSetIOCudaStreams` bound to this stream): the
|
/// only (the direct-NVENC submit path with `NvEncSetIOCudaStreams` bound to this stream): the
|
||||||
/// stream, not the CPU, orders completion, so the SOURCE must stay valid until the downstream
|
/// stream, not the CPU, orders completion, so the SOURCE must stay valid until the downstream
|
||||||
/// stream work (the encode) has finished.
|
/// stream work (the encode) has finished.
|
||||||
unsafe fn copy_async(copy: &CUDA_MEMCPY2D, what: &str) -> Result<()> {
|
unsafe fn copy_async(copy: &CUDA_MEMCPY2D, what: &str) -> Result<()> {
|
||||||
ck(cuMemcpy2DAsync_v2(copy, copy_stream()), what)
|
// SAFETY: caller contract: the shared context is current and `copy` describes live, in-bounds
|
||||||
|
// device/host memory that stays valid until the stream work completes (each caller carries that
|
||||||
|
// proof). Wrapper -> live table.
|
||||||
|
unsafe { ck(cuMemcpy2DAsync_v2(copy, copy_stream()), what) }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Block until everything enqueued on THIS THREAD's copy stream completed — the shared tail of
|
/// Block until everything enqueued on THIS THREAD's copy stream completed — the shared tail of
|
||||||
@@ -268,18 +276,25 @@ unsafe fn copy_async(copy: &CUDA_MEMCPY2D, what: &str) -> Result<()> {
|
|||||||
/// game's GPU load each exposed wait eats scheduling latency). The shared context must be
|
/// game's GPU load each exposed wait eats scheduling latency). The shared context must be
|
||||||
/// current.
|
/// current.
|
||||||
unsafe fn sync_copy_stream() -> Result<()> {
|
unsafe fn sync_copy_stream() -> Result<()> {
|
||||||
ck(cuStreamSynchronize(copy_stream()), "cuStreamSynchronize")
|
// SAFETY: caller contract: the shared context is current. Wrapper -> live table; synchronizing
|
||||||
|
// the dedicated copy stream touches no Rust memory.
|
||||||
|
unsafe { ck(cuStreamSynchronize(copy_stream()), "cuStreamSynchronize") }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `copy_blocking` when `sync`, else `copy_async` — the shared tail of the public `copy_*_to_device`
|
/// `copy_blocking` when `sync`, else `copy_async` — the shared tail of the public `copy_*_to_device`
|
||||||
/// helpers, whose `sync: false` mode carries `copy_async`'s source-lifetime contract.
|
/// helpers, whose `sync: false` mode carries `copy_async`'s source-lifetime contract.
|
||||||
unsafe fn copy_issue(copy: &CUDA_MEMCPY2D, what: &str, sync: bool) -> Result<()> {
|
unsafe fn copy_issue(copy: &CUDA_MEMCPY2D, what: &str, sync: bool) -> Result<()> {
|
||||||
|
// SAFETY: caller contract: the shared context is current and `copy` describes live, in-bounds
|
||||||
|
// memory (each caller carries that proof). The stream handle is the once-created per-process
|
||||||
|
// copy stream. Wrapper -> live table.
|
||||||
|
unsafe {
|
||||||
if sync {
|
if sync {
|
||||||
copy_blocking(copy, what)
|
copy_blocking(copy, what)
|
||||||
} else {
|
} else {
|
||||||
copy_async(copy, what)
|
copy_async(copy, what)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The calling thread's copy/launch stream as a raw handle, for binding external stream-ordering
|
/// The calling thread's copy/launch stream as a raw handle, for binding external stream-ordering
|
||||||
/// (the direct-NVENC `NvEncSetIOCudaStreams` hookup). Null = the NULL stream (priority-stream
|
/// (the direct-NVENC `NvEncSetIOCudaStreams` hookup). Null = the NULL stream (priority-stream
|
||||||
@@ -785,6 +800,10 @@ impl RegisteredTexture {
|
|||||||
/// The GL context and the shared CUDA context must both be current on this thread, and
|
/// The GL context and the shared CUDA context must both be current on this thread, and
|
||||||
/// `texture` must be a valid `GL_TEXTURE_2D`.
|
/// `texture` must be a valid `GL_TEXTURE_2D`.
|
||||||
pub unsafe fn register_gl(texture: u32) -> Result<RegisteredTexture> {
|
pub unsafe fn register_gl(texture: u32) -> Result<RegisteredTexture> {
|
||||||
|
// SAFETY: caller contract: the GL context owning `texture` and the shared CUDA context are
|
||||||
|
// current on this thread, and `texture` is a live, complete GL texture. The out-param is a
|
||||||
|
// live stack local; wrapper -> live table.
|
||||||
|
unsafe {
|
||||||
const GL_TEXTURE_2D: c_uint = 0x0DE1;
|
const GL_TEXTURE_2D: c_uint = 0x0DE1;
|
||||||
const CU_GRAPHICS_REGISTER_FLAGS_READ_ONLY: c_uint = 0x01;
|
const CU_GRAPHICS_REGISTER_FLAGS_READ_ONLY: c_uint = 0x01;
|
||||||
let mut resource: CUgraphicsResource = std::ptr::null_mut();
|
let mut resource: CUgraphicsResource = std::ptr::null_mut();
|
||||||
@@ -799,6 +818,7 @@ impl RegisteredTexture {
|
|||||||
)?;
|
)?;
|
||||||
Ok(RegisteredTexture { resource })
|
Ok(RegisteredTexture { resource })
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Map the texture for this frame, copy its (already-linear RGBA8) array into `dst`, then
|
/// Map the texture for this frame, copy its (already-linear RGBA8) array into `dst`, then
|
||||||
/// unmap. The copy is synchronized (on our priority stream) before unmap so `dst` is ready
|
/// unmap. The copy is synchronized (on our priority stream) before unmap so `dst` is ready
|
||||||
|
|||||||
@@ -307,13 +307,19 @@ pub(crate) fn cuda_api() -> Option<&'static CudaApi> {
|
|||||||
// present; every other entry runs after `context()` succeeded, so its wrapper always hits `Some`.
|
// present; every other entry runs after `context()` succeeded, so its wrapper always hits `Some`.
|
||||||
pub(crate) unsafe fn cuInit(flags: c_uint) -> CUresult {
|
pub(crate) unsafe fn cuInit(flags: c_uint) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuInit)(flags),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuInit)(flags) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuDeviceGet(device: *mut CUdevice, ordinal: c_int) -> CUresult {
|
pub(crate) unsafe fn cuDeviceGet(device: *mut CUdevice, ordinal: c_int) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuDeviceGet)(device, ordinal),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuDeviceGet)(device, ordinal) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -323,19 +329,28 @@ pub(crate) unsafe fn cuCtxCreate_v2(
|
|||||||
dev: CUdevice,
|
dev: CUdevice,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuCtxCreate_v2)(pctx, flags, dev),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuCtxCreate_v2)(pctx, flags, dev) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuCtxDestroy_v2(ctx: CUcontext) -> CUresult {
|
pub(crate) unsafe fn cuCtxDestroy_v2(ctx: CUcontext) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuCtxDestroy_v2)(ctx),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuCtxDestroy_v2)(ctx) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuCtxSetCurrent(ctx: CUcontext) -> CUresult {
|
pub(crate) unsafe fn cuCtxSetCurrent(ctx: CUcontext) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuCtxSetCurrent)(ctx),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuCtxSetCurrent)(ctx) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -347,25 +362,39 @@ pub(crate) unsafe fn cuMemAllocPitch_v2(
|
|||||||
element_size: c_uint,
|
element_size: c_uint,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuMemAllocPitch_v2)(dptr, pitch, width_bytes, height, element_size),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe {
|
||||||
|
(a.cuMemAllocPitch_v2)(dptr, pitch, width_bytes, height, element_size)
|
||||||
|
},
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuMemFree_v2(dptr: CUdeviceptr) -> CUresult {
|
pub(crate) unsafe fn cuMemFree_v2(dptr: CUdeviceptr) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuMemFree_v2)(dptr),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuMemFree_v2)(dptr) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuMemcpy2DAsync_v2(copy: *const CUDA_MEMCPY2D, stream: CUstream) -> CUresult {
|
pub(crate) unsafe fn cuMemcpy2DAsync_v2(copy: *const CUDA_MEMCPY2D, stream: CUstream) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuMemcpy2DAsync_v2)(copy, stream),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuMemcpy2DAsync_v2)(copy, stream) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuStreamSynchronize(stream: CUstream) -> CUresult {
|
pub(crate) unsafe fn cuStreamSynchronize(stream: CUstream) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuStreamSynchronize)(stream),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuStreamSynchronize)(stream) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -374,7 +403,10 @@ pub(crate) unsafe fn cuCtxGetStreamPriorityRange(
|
|||||||
greatest: *mut c_int,
|
greatest: *mut c_int,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuCtxGetStreamPriorityRange)(least, greatest),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuCtxGetStreamPriorityRange)(least, greatest) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -384,7 +416,10 @@ pub(crate) unsafe fn cuStreamCreateWithPriority(
|
|||||||
priority: c_int,
|
priority: c_int,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuStreamCreateWithPriority)(stream, flags, priority),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuStreamCreateWithPriority)(stream, flags, priority) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -395,7 +430,10 @@ pub(crate) unsafe fn cuGraphicsGLRegisterImage(
|
|||||||
flags: c_uint,
|
flags: c_uint,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuGraphicsGLRegisterImage)(resource, texture, target, flags),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuGraphicsGLRegisterImage)(resource, texture, target, flags) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -405,7 +443,10 @@ pub(crate) unsafe fn cuGraphicsMapResources(
|
|||||||
stream: *mut c_void,
|
stream: *mut c_void,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuGraphicsMapResources)(count, resources, stream),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuGraphicsMapResources)(count, resources, stream) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -415,7 +456,10 @@ pub(crate) unsafe fn cuGraphicsUnmapResources(
|
|||||||
stream: *mut c_void,
|
stream: *mut c_void,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuGraphicsUnmapResources)(count, resources, stream),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuGraphicsUnmapResources)(count, resources, stream) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -426,13 +470,21 @@ pub(crate) unsafe fn cuGraphicsSubResourceGetMappedArray(
|
|||||||
mip_level: c_uint,
|
mip_level: c_uint,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuGraphicsSubResourceGetMappedArray)(array, resource, array_index, mip_level),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe {
|
||||||
|
(a.cuGraphicsSubResourceGetMappedArray)(array, resource, array_index, mip_level)
|
||||||
|
},
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuGraphicsUnregisterResource(resource: CUgraphicsResource) -> CUresult {
|
pub(crate) unsafe fn cuGraphicsUnregisterResource(resource: CUgraphicsResource) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuGraphicsUnregisterResource)(resource),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuGraphicsUnregisterResource)(resource) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -441,7 +493,10 @@ pub(crate) unsafe fn cuImportExternalMemory(
|
|||||||
mem_handle_desc: *const CUDA_EXTERNAL_MEMORY_HANDLE_DESC,
|
mem_handle_desc: *const CUDA_EXTERNAL_MEMORY_HANDLE_DESC,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuImportExternalMemory)(ext_mem_out, mem_handle_desc),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuImportExternalMemory)(ext_mem_out, mem_handle_desc) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -451,13 +506,19 @@ pub(crate) unsafe fn cuExternalMemoryGetMappedBuffer(
|
|||||||
buffer_desc: *const CUDA_EXTERNAL_MEMORY_BUFFER_DESC,
|
buffer_desc: *const CUDA_EXTERNAL_MEMORY_BUFFER_DESC,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuExternalMemoryGetMappedBuffer)(dev_ptr, ext_mem, buffer_desc),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuExternalMemoryGetMappedBuffer)(dev_ptr, ext_mem, buffer_desc) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuDestroyExternalMemory(ext_mem: CUexternalMemory) -> CUresult {
|
pub(crate) unsafe fn cuDestroyExternalMemory(ext_mem: CUexternalMemory) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuDestroyExternalMemory)(ext_mem),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuDestroyExternalMemory)(ext_mem) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -466,13 +527,19 @@ pub(crate) unsafe fn cuImportExternalSemaphore(
|
|||||||
sem_handle_desc: *const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC,
|
sem_handle_desc: *const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuImportExternalSemaphore)(ext_sem_out, sem_handle_desc),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuImportExternalSemaphore)(ext_sem_out, sem_handle_desc) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuDestroyExternalSemaphore(ext_sem: CUexternalSemaphore) -> CUresult {
|
pub(crate) unsafe fn cuDestroyExternalSemaphore(ext_sem: CUexternalSemaphore) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuDestroyExternalSemaphore)(ext_sem),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuDestroyExternalSemaphore)(ext_sem) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -483,7 +550,10 @@ pub(crate) unsafe fn cuSignalExternalSemaphoresAsync(
|
|||||||
stream: CUstream,
|
stream: CUstream,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuSignalExternalSemaphoresAsync)(ext_sems, params, count, stream),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuSignalExternalSemaphoresAsync)(ext_sems, params, count, stream) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -494,13 +564,19 @@ pub(crate) unsafe fn cuWaitExternalSemaphoresAsync(
|
|||||||
stream: CUstream,
|
stream: CUstream,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuWaitExternalSemaphoresAsync)(ext_sems, params, count, stream),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuWaitExternalSemaphoresAsync)(ext_sems, params, count, stream) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuIpcGetMemHandle(handle: *mut CUipcMemHandle, dptr: CUdeviceptr) -> CUresult {
|
pub(crate) unsafe fn cuIpcGetMemHandle(handle: *mut CUipcMemHandle, dptr: CUdeviceptr) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuIpcGetMemHandle)(handle, dptr),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuIpcGetMemHandle)(handle, dptr) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -510,13 +586,19 @@ pub(crate) unsafe fn cuIpcOpenMemHandle(
|
|||||||
flags: c_uint,
|
flags: c_uint,
|
||||||
) -> CUresult {
|
) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuIpcOpenMemHandle)(dptr, handle, flags),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuIpcOpenMemHandle)(dptr, handle, flags) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub(crate) unsafe fn cuIpcCloseMemHandle(dptr: CUdeviceptr) -> CUresult {
|
pub(crate) unsafe fn cuIpcCloseMemHandle(dptr: CUdeviceptr) -> CUresult {
|
||||||
match cuda_api() {
|
match cuda_api() {
|
||||||
Some(a) => (a.cuIpcCloseMemHandle)(dptr),
|
// SAFETY: forwards this unsafe wrapper's arguments to the live dlopen'd entry point (the
|
||||||
|
// table is never unloaded — `mem::forget(lib)`); the caller upholds the driver-API contract,
|
||||||
|
// with the site-specific proof at each call site.
|
||||||
|
Some(a) => unsafe { (a.cuIpcCloseMemHandle)(dptr) },
|
||||||
None => CU_ERROR_NOT_LOADED,
|
None => CU_ERROR_NOT_LOADED,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -179,6 +179,10 @@ struct GlBlit {
|
|||||||
|
|
||||||
impl GlBlit {
|
impl GlBlit {
|
||||||
unsafe fn new(width: u32, height: u32) -> Result<GlBlit> {
|
unsafe fn new(width: u32, height: u32) -> Result<GlBlit> {
|
||||||
|
// SAFETY: caller contract (`import_inner`): the GL context and the shared CUDA context are
|
||||||
|
// current on this thread. Raw GL calls pass live locals whose pointers outlive each
|
||||||
|
// synchronous call; every created name is owned by `guard` until the struct exists.
|
||||||
|
unsafe {
|
||||||
// Declared before every GL name so it drops LAST on unwind — after the CUDA registration
|
// Declared before every GL name so it drops LAST on unwind — after the CUDA registration
|
||||||
// (declared below) has unregistered itself.
|
// (declared below) has unregistered itself.
|
||||||
let mut guard = GlNameGuard::default();
|
let mut guard = GlNameGuard::default();
|
||||||
@@ -239,11 +243,15 @@ impl GlBlit {
|
|||||||
pool,
|
pool,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Bind `image` to the source texture and render it into `dst_tex`.
|
/// Bind `image` to the source texture and render it into `dst_tex`.
|
||||||
///
|
///
|
||||||
/// # Safety: the GL context is current on this thread; `image` is a valid `EGLImage`.
|
/// # Safety: the GL context is current on this thread; `image` is a valid `EGLImage`.
|
||||||
unsafe fn run(&self, egl_image_target: EglImageTargetFn, image: *mut c_void) -> Result<()> {
|
unsafe fn run(&self, egl_image_target: EglImageTargetFn, image: *mut c_void) -> Result<()> {
|
||||||
|
// SAFETY: caller contract (`# Safety` above): GL context current, `image` a valid EGLImage.
|
||||||
|
// Raw GL calls pass names owned by `self`, created on this same context.
|
||||||
|
unsafe {
|
||||||
glBindTexture(GL_TEXTURE_2D, self.src_tex);
|
glBindTexture(GL_TEXTURE_2D, self.src_tex);
|
||||||
let _ = glGetError();
|
let _ = glGetError();
|
||||||
egl_image_target(GL_TEXTURE_2D, image);
|
egl_image_target(GL_TEXTURE_2D, image);
|
||||||
@@ -264,6 +272,7 @@ impl GlBlit {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Drop for GlBlit {
|
impl Drop for GlBlit {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
@@ -319,6 +328,10 @@ struct Nv12Blit {
|
|||||||
|
|
||||||
impl Nv12Blit {
|
impl Nv12Blit {
|
||||||
unsafe fn new(width: u32, height: u32) -> Result<Nv12Blit> {
|
unsafe fn new(width: u32, height: u32) -> Result<Nv12Blit> {
|
||||||
|
// SAFETY: caller contract (`import_inner`): the GL context and the shared CUDA context are
|
||||||
|
// current on this thread. Raw GL calls pass live locals whose pointers outlive each
|
||||||
|
// synchronous call; every created name is owned by `guard` until the struct exists.
|
||||||
|
unsafe {
|
||||||
ensure!(
|
ensure!(
|
||||||
width % 2 == 0 && height % 2 == 0,
|
width % 2 == 0 && height % 2 == 0,
|
||||||
"NV12 convert needs even dimensions (got {width}x{height})"
|
"NV12 convert needs even dimensions (got {width}x{height})"
|
||||||
@@ -403,11 +416,15 @@ impl Nv12Blit {
|
|||||||
test_src_storage: false,
|
test_src_storage: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Bind `image` to the source texture and run both convert passes into `y_tex`/`uv_tex`.
|
/// Bind `image` to the source texture and run both convert passes into `y_tex`/`uv_tex`.
|
||||||
///
|
///
|
||||||
/// # Safety: the GL context is current on this thread; `image` is a valid `EGLImage`.
|
/// # Safety: the GL context is current on this thread; `image` is a valid `EGLImage`.
|
||||||
unsafe fn run(&self, egl_image_target: EglImageTargetFn, image: *mut c_void) -> Result<()> {
|
unsafe fn run(&self, egl_image_target: EglImageTargetFn, image: *mut c_void) -> Result<()> {
|
||||||
|
// SAFETY: caller contract (`# Safety` above): GL context current, `image` a valid EGLImage.
|
||||||
|
// Raw GL calls pass names owned by `self`, created on this same context.
|
||||||
|
unsafe {
|
||||||
glBindTexture(GL_TEXTURE_2D, self.src_tex);
|
glBindTexture(GL_TEXTURE_2D, self.src_tex);
|
||||||
let _ = glGetError();
|
let _ = glGetError();
|
||||||
egl_image_target(GL_TEXTURE_2D, image);
|
egl_image_target(GL_TEXTURE_2D, image);
|
||||||
@@ -416,12 +433,16 @@ impl Nv12Blit {
|
|||||||
ensure!(e == 0, "glEGLImageTargetTexture2DOES failed ({e:#x})");
|
ensure!(e == 0, "glEGLImageTargetTexture2DOES failed ({e:#x})");
|
||||||
self.run_passes()
|
self.run_passes()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Run the two convert passes from whatever is currently in `src_tex` (caller populated it).
|
/// Run the two convert passes from whatever is currently in `src_tex` (caller populated it).
|
||||||
/// Shared by [`run`](Self::run) (EGLImage source) and the self-test (uploaded RGBA source).
|
/// Shared by [`run`](Self::run) (EGLImage source) and the self-test (uploaded RGBA source).
|
||||||
///
|
///
|
||||||
/// # Safety: the GL context is current on this thread.
|
/// # Safety: the GL context is current on this thread.
|
||||||
unsafe fn run_passes(&self) -> Result<()> {
|
unsafe fn run_passes(&self) -> Result<()> {
|
||||||
|
// SAFETY: caller contract (`# Safety` above): GL context current. Raw GL calls pass names
|
||||||
|
// owned by `self`, created on this same context.
|
||||||
|
unsafe {
|
||||||
glActiveTexture(GL_TEXTURE0);
|
glActiveTexture(GL_TEXTURE0);
|
||||||
glBindVertexArray(self.vao);
|
glBindVertexArray(self.vao);
|
||||||
// Y pass: full-res into the R8 target.
|
// Y pass: full-res into the R8 target.
|
||||||
@@ -443,6 +464,7 @@ impl Nv12Blit {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Drop for Nv12Blit {
|
impl Drop for Nv12Blit {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
@@ -498,11 +520,16 @@ struct Yuv444Blit {
|
|||||||
|
|
||||||
impl Yuv444Blit {
|
impl Yuv444Blit {
|
||||||
unsafe fn new(width: u32, height: u32) -> Result<Yuv444Blit> {
|
unsafe fn new(width: u32, height: u32) -> Result<Yuv444Blit> {
|
||||||
|
// SAFETY: caller contract (`import_inner`): the GL context and the shared CUDA context are
|
||||||
|
// current on this thread. Raw GL calls pass live locals whose pointers outlive each
|
||||||
|
// synchronous call; every created name is owned by `guard` until the struct exists.
|
||||||
|
unsafe {
|
||||||
ensure!(
|
ensure!(
|
||||||
width % 2 == 0 && height % 2 == 0,
|
width % 2 == 0 && height % 2 == 0,
|
||||||
"YUV444 convert needs even dimensions (got {width}x{height})"
|
"YUV444 convert needs even dimensions (got {width}x{height})"
|
||||||
);
|
);
|
||||||
let full_range = std::env::var("PUNKTFUNK_444_FULLRANGE").is_ok_and(|v| v.trim() == "1");
|
let full_range =
|
||||||
|
std::env::var("PUNKTFUNK_444_FULLRANGE").is_ok_and(|v| v.trim() == "1");
|
||||||
let (y_src, u_src, v_src) = yuv444_frag_sources(full_range);
|
let (y_src, u_src, v_src) = yuv444_frag_sources(full_range);
|
||||||
// Declared before every GL name so it drops LAST on unwind — after the CUDA registrations
|
// Declared before every GL name so it drops LAST on unwind — after the CUDA registrations
|
||||||
// (declared below) have unregistered themselves.
|
// (declared below) have unregistered themselves.
|
||||||
@@ -568,11 +595,15 @@ impl Yuv444Blit {
|
|||||||
pool,
|
pool,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Bind `image` to the source texture and run the three plane passes.
|
/// Bind `image` to the source texture and run the three plane passes.
|
||||||
///
|
///
|
||||||
/// # Safety: the GL context is current on this thread; `image` is a valid `EGLImage`.
|
/// # Safety: the GL context is current on this thread; `image` is a valid `EGLImage`.
|
||||||
unsafe fn run(&self, egl_image_target: EglImageTargetFn, image: *mut c_void) -> Result<()> {
|
unsafe fn run(&self, egl_image_target: EglImageTargetFn, image: *mut c_void) -> Result<()> {
|
||||||
|
// SAFETY: caller contract (`# Safety` above): GL context current, `image` a valid EGLImage.
|
||||||
|
// Raw GL calls pass names owned by `self`, created on this same context.
|
||||||
|
unsafe {
|
||||||
glBindTexture(GL_TEXTURE_2D, self.src_tex);
|
glBindTexture(GL_TEXTURE_2D, self.src_tex);
|
||||||
let _ = glGetError();
|
let _ = glGetError();
|
||||||
egl_image_target(GL_TEXTURE_2D, image);
|
egl_image_target(GL_TEXTURE_2D, image);
|
||||||
@@ -594,6 +625,7 @@ impl Yuv444Blit {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Drop for Yuv444Blit {
|
impl Drop for Yuv444Blit {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
|
|||||||
@@ -148,6 +148,10 @@ pub(crate) fn yuv444_frag_sources(full_range: bool) -> (Vec<u8>, Vec<u8>, Vec<u8
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) unsafe fn compile_shader(kind: u32, src: &[u8]) -> Result<u32> {
|
pub(crate) unsafe fn compile_shader(kind: u32, src: &[u8]) -> Result<u32> {
|
||||||
|
// SAFETY: caller contract: the GL context is current on this thread. `src` is a live slice; its
|
||||||
|
// pointer/length locals outlive the synchronous `glShaderSource`; the shader name is deleted on
|
||||||
|
// the compile-failure path.
|
||||||
|
unsafe {
|
||||||
let sh = glCreateShader(kind);
|
let sh = glCreateShader(kind);
|
||||||
ensure!(sh != 0, "glCreateShader failed");
|
ensure!(sh != 0, "glCreateShader failed");
|
||||||
let ptr = src.as_ptr() as *const i8;
|
let ptr = src.as_ptr() as *const i8;
|
||||||
@@ -162,10 +166,14 @@ pub(crate) unsafe fn compile_shader(kind: u32, src: &[u8]) -> Result<u32> {
|
|||||||
}
|
}
|
||||||
Ok(sh)
|
Ok(sh)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Compile+link the fullscreen-triangle program with fragment source `frag` and bind its `image`
|
/// Compile+link the fullscreen-triangle program with fragment source `frag` and bind its `image`
|
||||||
/// sampler to texture unit 0.
|
/// sampler to texture unit 0.
|
||||||
pub(crate) unsafe fn compile_program_with(frag: &[u8]) -> Result<u32> {
|
pub(crate) unsafe fn compile_program_with(frag: &[u8]) -> Result<u32> {
|
||||||
|
// SAFETY: caller contract: the GL context is current on this thread. All names are created here
|
||||||
|
// and deleted on every failure path; `c"image"` is a valid NUL-terminated literal.
|
||||||
|
unsafe {
|
||||||
// Callers retry per frame, so every error path below must delete what it created — a leak
|
// Callers retry per frame, so every error path below must delete what it created — a leak
|
||||||
// here is unbounded, not one-shot.
|
// here is unbounded, not one-shot.
|
||||||
let vs = compile_shader(GL_VERTEX_SHADER, VERT_SRC)?;
|
let vs = compile_shader(GL_VERTEX_SHADER, VERT_SRC)?;
|
||||||
@@ -201,7 +209,10 @@ pub(crate) unsafe fn compile_program_with(frag: &[u8]) -> Result<u32> {
|
|||||||
glUseProgram(0);
|
glUseProgram(0);
|
||||||
Ok(prog)
|
Ok(prog)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) unsafe fn compile_program() -> Result<u32> {
|
pub(crate) unsafe fn compile_program() -> Result<u32> {
|
||||||
compile_program_with(FRAG_SRC)
|
// SAFETY: caller contract: the GL context is current on this thread (forwarded to
|
||||||
|
// `compile_program_with`).
|
||||||
|
unsafe { compile_program_with(FRAG_SRC) }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -797,6 +797,11 @@ impl VkSlotBlend {
|
|||||||
gx: u32,
|
gx: u32,
|
||||||
gy: u32,
|
gy: u32,
|
||||||
) -> Result<vk::CommandBuffer> {
|
) -> Result<vk::CommandBuffer> {
|
||||||
|
// SAFETY: caller contract (`# Safety` above): the slot's previous submission completed, so
|
||||||
|
// its command buffer is re-recordable. Handles are owned by `self` on this single thread;
|
||||||
|
// `bytes` reborrows `push` (repr(C), size_of::<Push>()) for the synchronous push-constant
|
||||||
|
// copy; builder infos are locals outliving each call.
|
||||||
|
unsafe {
|
||||||
let alloc = self
|
let alloc = self
|
||||||
.slots
|
.slots
|
||||||
.get(id)
|
.get(id)
|
||||||
@@ -866,6 +871,7 @@ impl VkSlotBlend {
|
|||||||
d.end_command_buffer(cmd).context("end blend cmd")?;
|
d.end_command_buffer(cmd).context("end blend cmd")?;
|
||||||
Ok(cmd)
|
Ok(cmd)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Blend the uploaded cursor into `slot` at `(ox, oy)`: record, submit, fence-wait. The
|
/// Blend the uploaded cursor into `slot` at `(ox, oy)`: record, submit, fence-wait. The
|
||||||
/// caller has CPU-synced its CUDA frame copy first; the fence wait makes the shader's writes
|
/// caller has CPU-synced its CUDA frame copy first; the fence wait makes the shader's writes
|
||||||
|
|||||||
@@ -292,6 +292,10 @@ impl VkBridge {
|
|||||||
|
|
||||||
/// Import `fd` (dup'd internally; Vulkan owns the dup) as a transfer-src buffer of `size`.
|
/// Import `fd` (dup'd internally; Vulkan owns the dup) as a transfer-src buffer of `size`.
|
||||||
unsafe fn import_src(&mut self, fd: i32, size: u64) -> Result<()> {
|
unsafe fn import_src(&mut self, fd: i32, size: u64) -> Result<()> {
|
||||||
|
// SAFETY: caller contract: single-threaded use of handles this bridge owns; every builder
|
||||||
|
// info is built from locals that outlive the synchronous call reading them, and every
|
||||||
|
// fallible step destroys what it created before returning (comments below).
|
||||||
|
unsafe {
|
||||||
use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd};
|
use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd};
|
||||||
let dup = libc::dup(fd);
|
let dup = libc::dup(fd);
|
||||||
if dup < 0 {
|
if dup < 0 {
|
||||||
@@ -313,7 +317,8 @@ impl VkBridge {
|
|||||||
// STORAGE so the NV12 compute CSC can read it as an SSBO (T2.5b); harmless
|
// STORAGE so the NV12 compute CSC can read it as an SSBO (T2.5b); harmless
|
||||||
// for the plain copy path.
|
// for the plain copy path.
|
||||||
.usage(
|
.usage(
|
||||||
vk::BufferUsageFlags::TRANSFER_SRC | vk::BufferUsageFlags::STORAGE_BUFFER,
|
vk::BufferUsageFlags::TRANSFER_SRC
|
||||||
|
| vk::BufferUsageFlags::STORAGE_BUFFER,
|
||||||
)
|
)
|
||||||
.push_next(&mut ext_info),
|
.push_next(&mut ext_info),
|
||||||
None,
|
None,
|
||||||
@@ -377,9 +382,14 @@ impl VkBridge {
|
|||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// (Re)create the exportable destination of at least `size` bytes + its CUDA mapping.
|
/// (Re)create the exportable destination of at least `size` bytes + its CUDA mapping.
|
||||||
unsafe fn ensure_dst(&mut self, size: u64) -> Result<()> {
|
unsafe fn ensure_dst(&mut self, size: u64) -> Result<()> {
|
||||||
|
// SAFETY: caller contract: single-threaded use of handles this bridge owns; every builder
|
||||||
|
// info is built from locals that outlive the synchronous call reading them; created handles
|
||||||
|
// are destroyed on the error paths below or owned by `DstBuf`.
|
||||||
|
unsafe {
|
||||||
if self.dst.as_ref().is_some_and(|d| d.size >= size) {
|
if self.dst.as_ref().is_some_and(|d| d.size >= size) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -397,15 +407,17 @@ impl VkBridge {
|
|||||||
.size(size)
|
.size(size)
|
||||||
// STORAGE so the NV12 compute CSC can write it as an SSBO (T2.5b).
|
// STORAGE so the NV12 compute CSC can write it as an SSBO (T2.5b).
|
||||||
.usage(
|
.usage(
|
||||||
vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::STORAGE_BUFFER,
|
vk::BufferUsageFlags::TRANSFER_DST
|
||||||
|
| vk::BufferUsageFlags::STORAGE_BUFFER,
|
||||||
)
|
)
|
||||||
.push_next(&mut ext_info),
|
.push_next(&mut ext_info),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
.context("create export buffer")?;
|
.context("create export buffer")?;
|
||||||
let reqs = self.device.get_buffer_memory_requirements(buffer);
|
let reqs = self.device.get_buffer_memory_requirements(buffer);
|
||||||
let mem_type =
|
let mem_type = match self
|
||||||
match self.memory_type(reqs.memory_type_bits, vk::MemoryPropertyFlags::DEVICE_LOCAL) {
|
.memory_type(reqs.memory_type_bits, vk::MemoryPropertyFlags::DEVICE_LOCAL)
|
||||||
|
{
|
||||||
Ok(t) => t,
|
Ok(t) => t,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
self.device.destroy_buffer(buffer, None);
|
self.device.destroy_buffer(buffer, None);
|
||||||
@@ -471,12 +483,17 @@ impl VkBridge {
|
|||||||
});
|
});
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the RGB→NV12 compute pipeline once (T2.5b): two-SSBO descriptor set + a 28-byte
|
/// Build the RGB→NV12 compute pipeline once (T2.5b): two-SSBO descriptor set + a 28-byte
|
||||||
/// push-constant block matching `rgb2nv12_buf.comp`'s `Push`. A mid-build failure destroys
|
/// push-constant block matching `rgb2nv12_buf.comp`'s `Push`. A mid-build failure destroys
|
||||||
/// what was already created (the caller retries per frame, so a leak would be unbounded) and
|
/// what was already created (the caller retries per frame, so a leak would be unbounded) and
|
||||||
/// leaves `self.csc` `None`.
|
/// leaves `self.csc` `None`.
|
||||||
unsafe fn ensure_csc(&mut self) -> Result<()> {
|
unsafe fn ensure_csc(&mut self) -> Result<()> {
|
||||||
|
// SAFETY: caller contract: single-threaded use of handles this bridge owns. `build_csc`
|
||||||
|
// fills `csc` front to back; on failure the destroy calls below run in reverse creation
|
||||||
|
// order, and Vulkan destroys are defined no-ops on the null handles a partial build left.
|
||||||
|
unsafe {
|
||||||
if self.csc.is_some() {
|
if self.csc.is_some() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -500,13 +517,20 @@ impl VkBridge {
|
|||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
self.csc = Some(csc);
|
self.csc = Some(csc);
|
||||||
tracing::info!("Vulkan-bridge NV12 compute CSC ready (LINEAR path feeds NVENC native YUV)");
|
tracing::info!(
|
||||||
|
"Vulkan-bridge NV12 compute CSC ready (LINEAR path feeds NVENC native YUV)"
|
||||||
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The fallible half of [`ensure_csc`](Self::ensure_csc): fills `csc` front to back so the
|
/// The fallible half of [`ensure_csc`](Self::ensure_csc): fills `csc` front to back so the
|
||||||
/// caller knows exactly what to destroy when a step fails.
|
/// caller knows exactly what to destroy when a step fails.
|
||||||
unsafe fn build_csc(&mut self, csc: &mut Csc) -> Result<()> {
|
unsafe fn build_csc(&mut self, csc: &mut Csc) -> Result<()> {
|
||||||
|
// SAFETY: caller contract (via `ensure_csc`): single-threaded use of handles this bridge
|
||||||
|
// owns; every builder info is built from locals that outlive the synchronous call reading
|
||||||
|
// them; partial results land in `csc` for the caller to destroy on failure.
|
||||||
|
unsafe {
|
||||||
let words: Vec<u32> = CSC_SPV
|
let words: Vec<u32> = CSC_SPV
|
||||||
.chunks_exact(4)
|
.chunks_exact(4)
|
||||||
.map(|c| u32::from_le_bytes(c.try_into().unwrap()))
|
.map(|c| u32::from_le_bytes(c.try_into().unwrap()))
|
||||||
@@ -584,6 +608,7 @@ impl VkBridge {
|
|||||||
.context("allocate CSC descriptor set")?[0];
|
.context("allocate CSC descriptor set")?[0];
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Bridge one LINEAR dmabuf frame into a pooled NV12 CUDA buffer (latency plan T2.5b):
|
/// Bridge one LINEAR dmabuf frame into a pooled NV12 CUDA buffer (latency plan T2.5b):
|
||||||
/// instead of the plain byte copy, the compute CSC reads the imported RGB texels and writes
|
/// instead of the plain byte copy, the compute CSC reads the imported RGB texels and writes
|
||||||
|
|||||||
@@ -10,7 +10,11 @@
|
|||||||
|
|
||||||
// Unsafe-proof program: every `unsafe {}` / `unsafe impl` must carry a `// SAFETY:` proof. Each
|
// Unsafe-proof program: every `unsafe {}` / `unsafe impl` must carry a `// SAFETY:` proof. Each
|
||||||
// file keeps its own `#![deny(...)]` too; this crate-root deny is the catch-all gate.
|
// file keeps its own `#![deny(...)]` too; this crate-root deny is the catch-all gate.
|
||||||
|
// `unsafe_op_in_unsafe_fn` closes the gap the clippy lint leaves: operations inside an
|
||||||
|
// `unsafe fn` body are not "unsafe blocks", so without it ~45 functions' worth of raw driver
|
||||||
|
// calls sat OUTSIDE the invariant this crate advertises.
|
||||||
#![deny(clippy::undocumented_unsafe_blocks)]
|
#![deny(clippy::undocumented_unsafe_blocks)]
|
||||||
|
#![deny(unsafe_op_in_unsafe_fn)]
|
||||||
|
|
||||||
/// Wait for a dmabuf's implicit read-ready fence (`DMA_BUF_IOCTL_EXPORT_SYNC_FILE` + poll).
|
/// Wait for a dmabuf's implicit read-ready fence (`DMA_BUF_IOCTL_EXPORT_SYNC_FILE` + poll).
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
|
|||||||
Reference in New Issue
Block a user