commit 66a9edb8985c92374cd010a981ec6a54ad8a90cd Author: Tommi Niemi Date: Mon Apr 6 20:38:10 2026 +0700 Initial extraction from isis: direct /dev/kfd GPU compute AMD RDNA3 GPU driver via raw KFD ioctls. No ROCm, no OpenCL. - Device discovery and topology queries - VRAM/GTT/Userptr memory with shared address space - PM4 compute queue and kernel dispatch - Hand-written RDNA3 ASM kernels: matmul (3100 GFLOP/s), matvec, superlinear - Tile-safe buffer padding for OOB protection - Event-based interrupt wait (no CPU polling) Co-Authored-By: Claude Opus 4.6 (1M context) diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..9353f29 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "kfd" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..1fb2938 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "kfd" +version = "0.1.0" +edition = "2021" +description = "Direct /dev/kfd GPU compute for AMD RDNA3 — no ROCm, no OpenCL, just ioctl" +license = "MIT OR Apache-2.0" +repository = "https://git.rotko.net/tommi/kfd" + +[dependencies] +libc = "0.2" + +[dev-dependencies] diff --git a/src/compute.rs b/src/compute.rs new file mode 100644 index 0000000..6d13fb9 --- /dev/null +++ b/src/compute.rs @@ -0,0 +1,67 @@ +//! compute dispatch primitives. +//! +//! GpuFuture: poll gpu signal for async completion. +//! ComputeState: what the ctm feels about its compute (interoception). +//! +//! the ctm doesn't know about gpus. it just calls the harness +//! and feels whether compute is fast or slow. + +use crate::memory::GpuBuffer; +use std::time::Instant; + +// ─── future ───────────────────────────────────────────────── + +/// async gpu completion. poll or block. +pub struct GpuFuture { + signal_ptr: *const u32, + expected: u32, + submitted_at: Instant, +} + +unsafe impl Send for GpuFuture {} + +impl GpuFuture { + pub fn new(signal: &GpuBuffer, expected: u32) -> Self { + GpuFuture { + signal_ptr: signal.cpu_ptr as *const u32, + expected, + submitted_at: Instant::now(), + } + } + + /// true if gpu finished. + pub fn poll(&self) -> bool { + let val = unsafe { self.signal_ptr.read_volatile() }; + val >= self.expected + } + + /// block until done. returns elapsed, or None on timeout. + pub fn wait(&self, timeout_us: u64) -> Option { + let start = Instant::now(); + loop { + if self.poll() { + return Some(self.submitted_at.elapsed()); + } + if start.elapsed().as_micros() as u64 > timeout_us { + return None; + } + std::hint::spin_loop(); + } + } +} + +// ─── interoception ────────────────────────────────────────── + +/// what the ctm feels about its compute. +/// read-only from organism side. harness updates it. +#[derive(Debug, Clone, Default)] +pub struct ComputeState { + /// rolling average dispatch latency (microseconds) + pub latency_us: f32, + /// true when gpu dispatch is in flight + pub gpu_busy: bool, + /// total dispatches + pub dispatch_count: u64, + /// faults (should be 0) + pub fault_count: u32, +} diff --git a/src/device.rs b/src/device.rs new file mode 100644 index 0000000..5080ff8 --- /dev/null +++ b/src/device.rs @@ -0,0 +1,1260 @@ +//! Heterogeneous compute via KFD (Kernel Fusion Driver). +//! +//! KFD manages both CPU and GPU as HSA compute agents. +//! One driver, unified memory, automatic dispatch. +//! +//! Node 0: CPU (32 cores, AVX-512, DDR5) +//! Node 1: GPU (64 SIMDs, wave32, 8GB VRAM) +//! +//! Small ops → CPU (lower latency, AVX-512) +//! Large matmuls → GPU (higher throughput, 64 SIMDs) +//! Shared address space — zero-copy between CPU and GPU. +//! +//! Reference: tinygrad/tinygrad/runtime/ops_amd.py + +use crate::memory::{GpuAllocator, GpuBuffer}; +use crate::queue::ComputeQueue; +use crate::dispatch::{CodeObject, GpuProgram, KernArgs, KernelEntry}; +use std::os::unix::io::RawFd; + +/// compiled rdna3 kernels (gfx1102), hand-written asm. +/// test_store: y[i] = 42.0 (minimal proof kernel) +const TEST_STORE_CO: &[u8] = include_bytes!("kernels/test_store.co"); +/// matvec: y = W*x + b (raw asm, no hidden opencl args) +const MATVEC_CO: &[u8] = include_bytes!("kernels/matvec_asm.co"); +/// matmul_blocked: TM=128 TT4x4 — best for M >= 1536 +const MATMUL_BLOCKED_CO: &[u8] = include_bytes!("kernels/matmul_blocked.co"); +/// matmul_small: TM=32 TT2x2 PLR — best for M < 1536 +const MATMUL_SMALL_CO: &[u8] = include_bytes!("kernels/matmul_small.co"); +/// superlinear_fwd: per-neuron batched matvec for NLM +/// Y[n*O+o] = B[n*O+o] + sum_k W[n*O*K+o*K+k] * X[n*K+k] +const SUPERLINEAR_CO: &[u8] = include_bytes!("kernels/superlinear.co"); + +// ─── Node properties ──────────────────────────────────────── + +/// Properties of a KFD compute node (CPU or GPU). +#[derive(Debug, Default, Clone)] +pub struct NodeProperties { + pub node_id: u32, + pub gpu_id: u32, + pub cpu_cores_count: u32, + pub simd_count: u32, + pub wave_front_size: u32, + pub gfx_target_version: u32, + pub max_waves_per_simd: u32, + pub array_count: u32, + pub cu_per_simd_array: u32, + pub drm_render_minor: u32, + pub cwsr_size: u32, + pub max_engine_clk: u32, + // Memory + pub mem_size_bytes: u64, + pub mem_width: u32, + pub mem_clk_max: u32, +} + +impl NodeProperties { + pub fn is_cpu(&self) -> bool { self.cpu_cores_count > 0 } + pub fn is_gpu(&self) -> bool { self.simd_count > 0 } + + /// Read properties from sysfs for a given node. + fn read(node_id: u32) -> std::io::Result { + let base = format!("/sys/devices/virtual/kfd/kfd/topology/nodes/{}", node_id); + let props_path = format!("{}/properties", base); + let gpu_id_path = format!("{}/gpu_id", base); + let mem_path = format!("{}/mem_banks/0/properties", base); + + let gpu_id: u32 = std::fs::read_to_string(&gpu_id_path) + .unwrap_or_default().trim().parse().unwrap_or(0); + + let props_text = std::fs::read_to_string(&props_path)?; + let mut p = NodeProperties { node_id, gpu_id, ..Default::default() }; + + for line in props_text.lines() { + let mut parts = line.split_whitespace(); + let key = parts.next().unwrap_or(""); + let val: u64 = parts.next().and_then(|v| v.parse().ok()).unwrap_or(0); + match key { + "cpu_cores_count" => p.cpu_cores_count = val as u32, + "simd_count" => p.simd_count = val as u32, + "wave_front_size" => p.wave_front_size = val as u32, + "gfx_target_version" => p.gfx_target_version = val as u32, + "cwsr_size" => p.cwsr_size = val as u32, + "max_waves_per_simd" => p.max_waves_per_simd = val as u32, + "array_count" => p.array_count = val as u32, + "cu_per_simd_array" => p.cu_per_simd_array = val as u32, + "drm_render_minor" => p.drm_render_minor = val as u32, + "max_engine_clk_ccompute" => p.max_engine_clk = val as u32, + _ => {} + } + } + + // Read memory bank + if let Ok(mem_text) = std::fs::read_to_string(&mem_path) { + for line in mem_text.lines() { + let mut parts = line.split_whitespace(); + let key = parts.next().unwrap_or(""); + let val: u64 = parts.next().and_then(|v| v.parse().ok()).unwrap_or(0); + match key { + "size_in_bytes" => p.mem_size_bytes = val, + "width" => p.mem_width = val as u32, + "mem_clk_max" => p.mem_clk_max = val as u32, + _ => {} + } + } + } + + Ok(p) + } +} + +// ─── Device specs & status ────────────────────────────────── + +/// Static device specifications. Readable without opening the device +/// (from sysfs topology). Use `DeviceSpecs::probe()` for zero-cost discovery. +#[derive(Debug, Clone)] +pub struct DeviceSpecs { + /// CPU node properties. + pub cpu: NodeProperties, + /// GPU node properties. + pub gpu: NodeProperties, + /// Total VRAM in bytes. + pub vram_total: u64, + /// VRAM bus width in bits (e.g. 256). + pub vram_width: u32, + /// VRAM max clock in MHz. + pub vram_clock_mhz: u32, + /// Peak VRAM bandwidth in GB/s (computed: width * clock * 2 / 8 / 1000). + pub vram_bandwidth_gbps: f32, + /// Total system RAM in bytes. + pub ram_total: u64, + /// GPU gfx target version (e.g. 0x110002 for gfx1102). + pub gfx_version: u32, + /// GPU compute units (simd_count * cu_per_simd_array, or simd_count if no CU info). + pub compute_units: u32, +} + +impl DeviceSpecs { + /// Probe device specs from sysfs. Does NOT open /dev/kfd — safe to call + /// even without permissions, just needs sysfs readable. + pub fn probe() -> std::io::Result { + let mut cpu = None; + let mut gpu = None; + for node in 0..16 { + match NodeProperties::read(node) { + Ok(p) if p.is_gpu() && gpu.is_none() => gpu = Some(p), + Ok(p) if p.is_cpu() && cpu.is_none() => cpu = Some(p), + _ => {} + } + } + let cpu = cpu.ok_or_else(|| std::io::Error::new( + std::io::ErrorKind::NotFound, "no CPU node in KFD topology"))?; + let gpu = gpu.ok_or_else(|| std::io::Error::new( + std::io::ErrorKind::NotFound, "no GPU node in KFD topology"))?; + + let vram_total = gpu.mem_size_bytes; + let vram_width = gpu.mem_width; + let vram_clock = gpu.mem_clk_max; + // DDR effective: clock * 2 (double data rate) + // Bandwidth = width_bits / 8 * effective_clock * 1e6 / 1e9 + let bw = (vram_width as f32 / 8.0) * (vram_clock as f32 * 2.0) / 1000.0; + + Ok(DeviceSpecs { + vram_total, + vram_width, + vram_clock_mhz: vram_clock, + vram_bandwidth_gbps: bw, + ram_total: cpu.mem_size_bytes, + gfx_version: gpu.gfx_target_version, + compute_units: if gpu.cu_per_simd_array > 0 { + gpu.simd_count * gpu.cu_per_simd_array + } else { + gpu.simd_count + }, + cpu, + gpu, + }) + } + + /// Human-readable summary. + pub fn summary(&self) -> String { + format!( + "GPU: gfx{:x}, {} CUs, {} GB VRAM (GDDR6-{} MHz, {}-bit, {:.0} GB/s)\n\ + CPU: {} cores, {} GB RAM", + self.gfx_version, + self.compute_units, + self.vram_total / (1024 * 1024 * 1024), + self.vram_clock_mhz, + self.vram_width, + self.vram_bandwidth_gbps, + self.cpu.cpu_cores_count, + self.ram_total / (1024 * 1024 * 1024), + ) + } + + /// Can this model's weights fit in VRAM? + /// `weight_bytes`: total weight size in bytes. + /// Returns (fits, headroom_bytes). + pub fn weights_fit(&self, weight_bytes: u64) -> (bool, i64) { + let headroom = self.vram_total as i64 - weight_bytes as i64; + (headroom > 0, headroom) + } +} + +/// Live device status. Requires an open KFD file descriptor. +#[derive(Debug, Clone)] +pub struct DeviceStatus { + /// Currently available (free) VRAM in bytes. + pub vram_available: u64, + /// Total VRAM in bytes (from specs). + pub vram_total: u64, + /// VRAM used = total - available. + pub vram_used: u64, + /// Current shader clock in MHz (from sysfs, None if unreadable). + pub sclk_mhz: Option, +} + +impl DeviceStatus { + /// Utilization ratio: 0.0 = empty, 1.0 = full. + pub fn vram_utilization(&self) -> f32 { + if self.vram_total == 0 { return 0.0; } + self.vram_used as f32 / self.vram_total as f32 + } + + /// Human-readable summary. + pub fn summary(&self) -> String { + let used_mb = self.vram_used / (1024 * 1024); + let total_mb = self.vram_total / (1024 * 1024); + let avail_mb = self.vram_available / (1024 * 1024); + let clk = self.sclk_mhz.map(|m| format!("{} MHz", m)) + .unwrap_or_else(|| "unknown".into()); + format!( + "VRAM: {} MB / {} MB ({} MB free, {:.0}% used), sclk: {}", + used_mb, total_mb, avail_mb, + self.vram_utilization() * 100.0, + clk, + ) + } +} + +// ─── Unified HSA device ───────────────────────────────────── + +/// Unified heterogeneous compute device via KFD. +/// +/// Manages both CPU and GPU nodes. Allocates shared memory visible +/// to both. Dispatches compute to the appropriate agent based on +/// problem size. +pub struct HsaDevice { + pub kfd_fd: RawFd, + pub drm_fd: RawFd, + pub cpu: NodeProperties, + pub gpu: NodeProperties, + pub alloc: GpuAllocator, + /// GPU resources — these fields drop BEFORE _fd_guard (declaration order). + _code_objects: Vec, + pub kernels: Option>, + pub queue: ComputeQueue, + pub signal: GpuBuffer, + pub event_page: GpuBuffer, + pub scratch: GpuBuffer, + signal_value: u32, + pub vram_available: u64, + pub queue_event_id: u32, + pub queue_event_mailbox_ptr: u64, + pub gpu_dispatch_threshold: usize, + /// MUST be last field: closes kfd_fd/drm_fd after all GpuBuffers are dropped. + _fd_guard: FdGuard, +} + +impl HsaDevice { + /// Open the HSA device. Discovers CPU + GPU from KFD topology. + pub fn open() -> std::io::Result { + let kfd_fd = unsafe { libc::open(b"/dev/kfd\0".as_ptr() as _, libc::O_RDWR) }; + if kfd_fd < 0 { + return Err(std::io::Error::new(std::io::ErrorKind::NotFound, + "/dev/kfd not available")); + } + + let (major, minor) = crate::ioctl::get_version(kfd_fd)?; + + // Discover all nodes + let mut cpu = None; + let mut gpu = None; + for node in 0..16 { + match NodeProperties::read(node) { + Ok(p) if p.is_gpu() && gpu.is_none() => gpu = Some(p), + Ok(p) if p.is_cpu() && cpu.is_none() => cpu = Some(p), + _ => {} + } + } + + let cpu = cpu.ok_or_else(|| std::io::Error::new( + std::io::ErrorKind::NotFound, "no CPU node in KFD topology"))?; + let gpu = gpu.ok_or_else(|| std::io::Error::new( + std::io::ErrorKind::NotFound, "no GPU node in KFD topology"))?; + + #[cfg(debug_assertions)] + eprintln!(" KFD {}.{} — HSA topology:", major, minor); + #[cfg(debug_assertions)] + eprintln!(" CPU: {} cores, {} MHz, {} GB RAM (DDR{}-{}, {}-bit)", + cpu.cpu_cores_count, cpu.max_engine_clk, + cpu.mem_size_bytes / (1024 * 1024 * 1024), + if cpu.mem_clk_max > 2400 { "5" } else { "4" }, + cpu.mem_clk_max, cpu.mem_width); + #[cfg(debug_assertions)] + eprintln!(" GPU: gfx{:x}, {} SIMDs, wave{}, {} GB VRAM (GDDR6-{}, {}-bit)", + gpu.gfx_target_version, gpu.simd_count, gpu.wave_front_size, + gpu.mem_size_bytes / (1024 * 1024 * 1024), + gpu.mem_clk_max, gpu.mem_width); + + // Open DRM render node for GPU + let drm_path = format!("/dev/dri/renderD{}\0", gpu.drm_render_minor); + let drm_fd = unsafe { libc::open(drm_path.as_ptr() as _, libc::O_RDWR) }; + if drm_fd < 0 { + unsafe { libc::close(kfd_fd); } + return Err(std::io::Error::last_os_error()); + } + + // Acquire VM for GPU + crate::ioctl::acquire_vm(kfd_fd, drm_fd, gpu.gpu_id) + .map_err(|e| { eprintln!(" acquire_vm failed: {}", e); e })?; + + // Runtime enable (required before queue creation on KFD >= 1.14) + if major > 1 || (major == 1 && minor >= 14) { + crate::ioctl::runtime_enable(kfd_fd) + .map_err(|e| { eprintln!(" runtime_enable failed: {}", e); e })?; + } + + let vram_available = crate::ioctl::available_memory(kfd_fd, gpu.gpu_id).unwrap_or(0); + + let alloc = GpuAllocator { + kfd_fd, + drm_fd, + gpu_id: gpu.gpu_id, + }; + + // Allocate event page (GTT, 0x8000 bytes) — must exist before CREATE_EVENT + let event_page = alloc.alloc_gtt(0x8000)?; + + // Create events before queue (KFD requirement) + // First event with event_page_offset=handle allocates the event page + let _first_event = crate::ioctl::create_event(kfd_fd, crate::ioctl::EVENT_SIGNAL, 0, + event_page.handle)?; + // Queue event (auto_reset=1) — used for interrupt signaling + let queue_event = crate::ioctl::create_event(kfd_fd, crate::ioctl::EVENT_SIGNAL, 1, 0)?; + let _mem_event = crate::ioctl::create_event(kfd_fd, crate::ioctl::EVENT_MEMORY, 0, 0)?; + let _hw_event = crate::ioctl::create_event(kfd_fd, crate::ioctl::EVENT_HW_EXCEPTION, 0, 0)?; + + // Compute mailbox pointer: event_page_va + event_slot_index * 8 + let queue_event_mailbox_ptr = event_page.va_addr + + queue_event.event_slot_index as u64 * 8; + + let queue = ComputeQueue::new(&alloc) + .map_err(|e| { eprintln!(" queue creation failed: {}", e); e })?; + // Scratch buffer for compute dispatch (2MB, VRAM) + let scratch = alloc.alloc_vram(2 * 1024 * 1024)?; + + // Signal buffer must be GPU-writable (flags 0xF0000004 matching tinygrad) + let signal = alloc.alloc_userptr_public(4096)?; + signal.write(0, &[0u8; 8]); + + // load compiled gpu kernels (raw asm) + // each code object owns its GpuBuffer — store them all to keep code in VRAM + let mut code_objects: Vec = Vec::new(); + for co_bytes in [TEST_STORE_CO, MATVEC_CO, MATMUL_BLOCKED_CO, MATMUL_SMALL_CO, SUPERLINEAR_CO] { + match CodeObject::load(&alloc, co_bytes) { + Ok(co) => code_objects.push(co), + Err(e) => eprintln!(" warning: kernel load failed: {}", e), + } + } + // merge kernel entries (they reference code_addr which stays valid while code_objects lives) + let mut kernel_map = std::collections::HashMap::new(); + for co in &code_objects { + for (name, entry) in &co.kernels { + kernel_map.insert(name.clone(), entry.clone()); + } + } + let kernels = if kernel_map.is_empty() { None } else { + #[cfg(debug_assertions)] + eprintln!(" kernels: {:?}", kernel_map.keys().collect::>()); + Some(kernel_map) + }; + + // GPU dispatch threshold: ~64K FLOPs + // Below this, CPU AVX-512 is faster due to dispatch overhead + let gpu_dispatch_threshold = 64 * 1024; + + // lock GPU clocks to max for consistent performance + let sysfs_device = format!("/sys/class/drm/renderD{}/device", gpu.drm_render_minor); + let clocks_locked = FdGuard::lock_clocks(&sysfs_device); + #[cfg(debug_assertions)] + if clocks_locked { + eprintln!(" gpu clocks: locked to max"); + } else { + eprintln!(" gpu clocks: auto (no sysfs write access)"); + } + + Ok(HsaDevice { + kfd_fd, + drm_fd, + cpu, + gpu, + alloc, + _code_objects: code_objects, + kernels, + queue, + signal, + event_page, + scratch, + signal_value: 0, + vram_available, + queue_event_id: queue_event.event_id, + queue_event_mailbox_ptr, + gpu_dispatch_threshold, + _fd_guard: FdGuard { + kfd_fd, drm_fd, + sysfs_device: if clocks_locked { Some(sysfs_device) } else { None }, + }, + }) + } + + /// Allocate a VRAM buffer and upload f32 data. + pub fn upload_f32(&self, data: &[f32]) -> std::io::Result { + // +4096: extra page for RDNA3 global_load_b128 cache line prefetch + // at allocation boundary (kernel prefetch reads 1 tile past last load) + let size = ((data.len() * 4 + 4096 + 4095) & !4095) as u64; + let buf = self.alloc.alloc_vram(size)?; + buf.write_f32(0, data); + Ok(buf) + } + + /// Upload a row-major matrix as column-major (transposed) in VRAM. + /// Input: data[rows][cols] row-major. Output: buf[cols][rows] in VRAM. + /// This gives coalesced access when threads read consecutive row indices + /// from the same column — critical for RDNA3's narrow 128-bit memory bus. + pub fn upload_f32_col_major(&self, data: &[f32], rows: usize, cols: usize) + -> std::io::Result + { + assert_eq!(data.len(), rows * cols); + let size = ((rows * cols * 4 + 4096 + 4095) & !4095) as u64; + let buf = self.alloc.alloc_vram(size)?; + let dst = unsafe { + std::slice::from_raw_parts_mut(buf.cpu_ptr as *mut f32, rows * cols) + }; + for r in 0..rows { + for c in 0..cols { + dst[c * rows + r] = data[r * cols + c]; + } + } + Ok(buf) + } + + /// Allocate a VRAM buffer of given size (bytes), zeroed. + pub fn alloc_output(&self, size_bytes: usize) -> std::io::Result { + let size = ((size_bytes + 4095) & !4095) as u64; + let buf = self.alloc.alloc_vram(size)?; + unsafe { std::ptr::write_bytes(buf.cpu_ptr, 0, size as usize); } + Ok(buf) + } + + /// dispatch a kernel and return a future (non-blocking). + pub fn dispatch_async(&mut self, name: &str, + kernargs: &GpuBuffer, + grid: [u32; 3], block: [u32; 3]) -> Option { + let entry = match self.kernels.as_ref().and_then(|m| m.get(name)) { + Some(e) => e.clone(), + None => return None, + }; + self.signal_value += 1; + self.queue.dispatch_lds( + entry.code_addr, + entry.desc.pgm_rsrc1, entry.desc.pgm_rsrc2, entry.desc.pgm_rsrc3, + kernargs.va_addr, self.scratch.va_addr, + grid, block, + entry.desc.group_segment_size, + ); + self.queue.signal(self.signal.va_addr, self.signal_value, + self.queue_event_mailbox_ptr, self.queue_event_id); + self.queue.submit(); + Some(crate::compute::GpuFuture::new(&self.signal, self.signal_value)) + } + + /// Queue a dispatch without signaling or submitting. + /// Call `submit_wait()` after all dispatches are queued. + pub fn dispatch_enqueue(&mut self, name: &str, + kernargs: &GpuBuffer, + grid: [u32; 3], block: [u32; 3]) -> bool { + let entry = match self.kernels.as_ref().and_then(|m| m.get(name)) { + Some(e) => e.clone(), + None => return false, + }; + self.queue.dispatch_lds( + entry.code_addr, + entry.desc.pgm_rsrc1, entry.desc.pgm_rsrc2, entry.desc.pgm_rsrc3, + kernargs.va_addr, self.scratch.va_addr, + grid, block, + entry.desc.group_segment_size, + ); + // CS_PARTIAL_FLUSH between dispatches (same as after dispatch in queue.rs) + true + } + + /// Signal + submit + wait via kfd event interrupt. + /// Call after dispatch_enqueue batch. Blocks until all dispatches complete. + pub fn submit_wait(&mut self, timeout_ms: u32) -> bool { + self.signal_value += 1; + self.queue.signal(self.signal.va_addr, self.signal_value, + self.queue_event_mailbox_ptr, self.queue_event_id); + self.queue.submit(); + self.wait_gpu(timeout_ms) + } + + /// Submit all queued packets and return a future for the last signal. + pub fn submit(&mut self) -> crate::compute::GpuFuture { + self.signal_value += 1; + self.queue.signal(self.signal.va_addr, self.signal_value, + self.queue_event_mailbox_ptr, self.queue_event_id); + self.queue.submit(); + crate::compute::GpuFuture::new(&self.signal, self.signal_value) + } + + /// Wait for the last signal via kfd event interrupt. + /// The GPU's RELEASE_MEM packet fires an interrupt when the signal + /// is written. The kfd driver wakes us — zero PCIe round-trips. + pub fn wait_gpu(&mut self, timeout_ms: u32) -> bool { + // fast path: check if already done (single volatile read) + let sig_ptr = self.signal.cpu_ptr as *const u32; + let val = unsafe { sig_ptr.read_volatile() }; + if val >= self.signal_value { return true; } + + let mut event = crate::ioctl::EventData::default(); + event.event_id = self.queue_event_id; + match crate::ioctl::wait_events(self.kfd_fd, &mut [event], false, timeout_ms) { + Ok(_) => { + let v = unsafe { sig_ptr.read_volatile() }; + if v < self.signal_value { + // event fired but signal not ready — brief spin + for _ in 0..10_000_000 { + let v2 = unsafe { sig_ptr.read_volatile() }; + if v2 >= self.signal_value { return true; } + std::hint::spin_loop(); + } + let v3 = unsafe { sig_ptr.read_volatile() }; + eprintln!("wait_gpu: event ok but signal={} < {}", v3, self.signal_value); + false + } else { + true + } + } + Err(e) => { + let val = unsafe { sig_ptr.read_volatile() }; + eprintln!("wait_gpu: err={} signal={} expected={}", e, val, self.signal_value); + val >= self.signal_value + } + } + } + + /// dispatch a kernel by name and wait for completion (blocking). + pub fn dispatch_kernel(&mut self, name: &str, + kernargs: &GpuBuffer, + grid: [u32; 3], block: [u32; 3]) -> bool { + let entry = match self.kernels.as_ref().and_then(|m| m.get(name)) { + Some(e) => e.clone(), + None => return false, + }; + self.signal_value += 1; + + // Record put position before dispatch for ring dump + let put_before = self.queue.put; + + self.queue.dispatch_lds( + entry.code_addr, + entry.desc.pgm_rsrc1, + entry.desc.pgm_rsrc2, + entry.desc.pgm_rsrc3, + kernargs.va_addr, + self.scratch.va_addr, + grid, block, + entry.desc.group_segment_size, + ); + self.queue.signal(self.signal.va_addr, self.signal_value, + self.queue_event_mailbox_ptr, self.queue_event_id); + + // Dump ring contents before submit (debug) + #[cfg(debug_assertions)] + { + let ring = self.queue.ring.as_slice::(); + let n = (self.queue.put - put_before) as usize; + eprintln!(" dispatch PM4 ({} DWORDs, code=0x{:x} kargs=0x{:x}):", + n, entry.code_addr, kernargs.va_addr); + for i in 0..n { + let idx = (put_before as usize + i) % (self.queue.ring.size as usize / 4); + eprintln!(" [{:3}] 0x{:08x}", i, ring[idx]); + } + } + + self.queue.submit(); + self.wait_gpu(10_000) + } + + /// Enqueue Y = X * W^T + B, selecting the optimal kernel for the shape. + /// W is row-major [M×K], X is row-major [N×K], Y is [N×M]. + /// Uses matmul_blocked (TM=128) for large M, matmul_small (TM=32) for small M. + /// + /// SAFETY: M, K, N must be padded to tile boundaries BEFORE calling this. + /// - M must be a multiple of 128 (matmul_blocked) or 32 (matmul_small) + /// - K must be a multiple of 8 + /// - N must be a multiple of 32 + /// - All buffers (W, B, X, Y) must be allocated for the padded dimensions. + /// Violating this will cause GPU OOB access → device hang → Xorg crash. + /// Use `dispatch_matmul_safe` for automatic padding. + pub fn dispatch_matmul_enqueue(&mut self, + w_row: &GpuBuffer, w_col: &GpuBuffer, + b: &GpuBuffer, x: &GpuBuffer, y: &GpuBuffer, + m: u32, k: u32, n: u32) -> bool { + // Guard: reject dimensions that would cause OOB + if k % 8 != 0 || n % 32 != 0 { + eprintln!("dispatch_matmul_enqueue: REJECTING unaligned dims M={} K={} N={}", m, k, n); + return false; + } + if m >= 1536 && m % 128 != 0 { + eprintln!("dispatch_matmul_enqueue: REJECTING M={} (not multiple of 128 for blocked)", m); + return false; + } + if m < 1536 && m % 32 != 0 { + eprintln!("dispatch_matmul_enqueue: REJECTING M={} (not multiple of 32 for small)", m); + return false; + } + + let (kernel, nwg) = if m >= 1536 { + ("matmul_blocked", ((m + 127) / 128) * ((n + 31) / 32)) + } else { + ("matmul_small", ((m + 31) / 32) * ((n + 31) / 32)) + }; + + let w = if m >= 1536 { w_row } else { w_col }; + + let mut args = KernArgs::new(); + args.push_ptr(w); args.push_ptr(b); + args.push_ptr(x); args.push_ptr(y); + args.push_u32(m); args.push_u32(k); args.push_u32(n); + let args_buf = match args.upload(&self.alloc) { + Ok(buf) => buf, + Err(_) => return false, + }; + + self.dispatch_enqueue(kernel, &args_buf, [nwg, 1, 1], [256, 1, 1]) + } + + /// Enqueue a SuperLinear forward: Y[n*O+o] = B[n*O+o] + sum_k W[..]*X[..]. + /// N=n_neurons, O=out_per, K=in_per. All buffers sized for N*O*K etc. + /// Returns false if kernel not loaded. + pub fn dispatch_superlinear_enqueue(&mut self, + w: &GpuBuffer, b: &GpuBuffer, + x: &GpuBuffer, y: &GpuBuffer, + n_neurons: u32, out_per: u32, in_per: u32) -> bool { + let total = n_neurons * out_per; + let nwg = (total + 255) / 256; + + let mut args = KernArgs::new(); + args.push_ptr(w); args.push_ptr(b); + args.push_ptr(x); args.push_ptr(y); + args.push_u32(n_neurons); args.push_u32(out_per); args.push_u32(in_per); + let args_buf = match args.upload(&self.alloc) { + Ok(buf) => buf, + Err(_) => return false, + }; + + self.dispatch_enqueue("superlinear_fwd", &args_buf, [nwg, 1, 1], [256, 1, 1]) + } + + /// Dispatch a GpuProgram directly and wait for completion. + pub fn dispatch_sync(&mut self, program: &GpuProgram, + kernargs: &GpuBuffer, + grid: [u32; 3], block: [u32; 3]) { + self.signal_value += 1; + program.dispatch(&mut self.queue, kernargs, self.scratch.va_addr, + grid, block, &self.signal, self.signal_value, + self.queue_event_mailbox_ptr, self.queue_event_id); + if !self.wait_gpu(10_000) { + eprintln!("WARNING: GPU dispatch timed out after 10s"); + } + } + + /// Load a code object into GPU memory. + pub fn load_program(&self, code_object: &[u8]) -> std::io::Result { + GpuProgram::load(&self.alloc, code_object) + } + + /// read current GPU shader clock in mhz from sysfs. + pub fn current_sclk_mhz(&self) -> Option { + let path = format!("/sys/class/drm/renderD{}/device/pp_dpm_sclk", + self.gpu.drm_render_minor); + let text = std::fs::read_to_string(&path).ok()?; + // active level marked with * + text.lines() + .find(|l| l.contains('*')) + .and_then(|l| { + // "2: 2330Mhz *" → extract 2330 + l.split(':').nth(1) + .and_then(|s| s.trim().split('M').next()) + .and_then(|s| s.trim().parse().ok()) + }) + } + + /// Should this operation go to GPU? Based on estimated FLOPs. + pub fn should_use_gpu(&self, flops: usize) -> bool { + flops >= self.gpu_dispatch_threshold + } + + /// Get static device specifications (from the already-read topology). + pub fn specs(&self) -> DeviceSpecs { + let vram_total = self.gpu.mem_size_bytes; + let vram_width = self.gpu.mem_width; + let vram_clock = self.gpu.mem_clk_max; + let bw = (vram_width as f32 / 8.0) * (vram_clock as f32 * 2.0) / 1000.0; + + DeviceSpecs { + vram_total, + vram_width, + vram_clock_mhz: vram_clock, + vram_bandwidth_gbps: bw, + ram_total: self.cpu.mem_size_bytes, + gfx_version: self.gpu.gfx_target_version, + compute_units: if self.gpu.cu_per_simd_array > 0 { + self.gpu.simd_count * self.gpu.cu_per_simd_array + } else { + self.gpu.simd_count + }, + cpu: self.cpu.clone(), + gpu: self.gpu.clone(), + } + } + + /// Query live device status (VRAM usage, clock speed). + pub fn status(&self) -> DeviceStatus { + let vram_available = crate::ioctl::available_memory(self.kfd_fd, self.gpu.gpu_id) + .unwrap_or(self.vram_available); + let vram_total = self.gpu.mem_size_bytes; + + DeviceStatus { + vram_available, + vram_total, + vram_used: vram_total.saturating_sub(vram_available), + sclk_mhz: self.current_sclk_mhz(), + } + } +} + +/// Guard that closes fds on drop. Placed as LAST field in HsaDevice +/// so it drops after all GpuBuffers (which need the fds for cleanup). +struct FdGuard { + kfd_fd: RawFd, + drm_fd: RawFd, + /// sysfs device path for clock control (e.g. /sys/class/drm/renderD128/device) + sysfs_device: Option, +} + +impl FdGuard { + /// Lock GPU to peak performance (max clocks, no DVFS). + fn lock_clocks(sysfs_device: &str) -> bool { + let perf_path = format!("{}/power_dpm_force_performance_level", sysfs_device); + // profile_peak forces max clocks immediately — no DVFS ramp needed + std::fs::write(&perf_path, "profile_peak").is_ok() + } + + fn unlock_clocks(sysfs_device: &str) { + let perf_path = format!("{}/power_dpm_force_performance_level", sysfs_device); + let _ = std::fs::write(&perf_path, "auto"); + } +} + +impl Drop for FdGuard { + fn drop(&mut self) { + if let Some(ref path) = self.sysfs_device { + FdGuard::unlock_clocks(path); + } + unsafe { + libc::close(self.drm_fd); + libc::close(self.kfd_fd); + } + } +} + +/// Check if KFD is available on this system. +pub fn is_available() -> bool { + std::path::Path::new("/dev/kfd").exists() +} + +// ─── Unified ComputeBackend ───────────────────────────────── + +// ─── Backend integration ─────────────────────────────────── +// HsaBackend and trait impls (ComputeBackend, TransformerOps) +// live in the downstream crate (e.g., isis) that depends on kfd. +// This crate only exposes HsaDevice for raw GPU access. + +// Backend trait implementations (ComputeBackend, TransformerOps) +// are defined in downstream crates that depend on `kfd`. +// See isis::host::device::kfd for the full integration. + +// ─── Tests ────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hsa_topology() { + if !is_available() { + eprintln!(" SKIP: /dev/kfd not available"); + return; + } + let dev = match HsaDevice::open() { + Ok(d) => d, + Err(e) => { eprintln!(" Cannot open: {}", e); return; } + }; + + assert!(dev.cpu.is_cpu()); + assert!(dev.gpu.is_gpu()); + eprintln!(" CPU: {} cores, {} GB", dev.cpu.cpu_cores_count, + dev.cpu.mem_size_bytes / (1024*1024*1024)); + eprintln!(" GPU: {} SIMDs, {} GB VRAM", + dev.gpu.simd_count, dev.gpu.mem_size_bytes / (1024*1024*1024)); + } + + #[test] + fn test_hsa_alloc_vram() { + if !is_available() { return; } + let dev = match HsaDevice::open() { + Ok(d) => d, + Err(_) => return, + }; + + let buf = dev.alloc.alloc_vram(4096).expect("VRAM alloc failed"); + let data: Vec = (0..256).map(|i| i as f32).collect(); + buf.write_f32(0, &data); + let readback = buf.read_f32(0, 256); + assert_eq!(data, readback); + eprintln!(" VRAM write/read: OK"); + } + + #[test] + fn test_hsa_alloc_gtt() { + if !is_available() { return; } + let dev = match HsaDevice::open() { + Ok(d) => d, + Err(_) => return, + }; + + let buf = dev.alloc.alloc_gtt(4096).expect("GTT alloc failed"); + buf.write(0, b"hello hsa"); + assert_eq!(&buf.read(0, 9), b"hello hsa"); + eprintln!(" GTT write/read: OK"); + } + + #[test] + fn test_hsa_queue() { + if !is_available() { return; } + let dev = match HsaDevice::open() { + Ok(d) => d, + Err(_) => return, + }; + eprintln!(" Queue ID: {}", dev.queue.queue_id); + } + + #[test] + fn test_hsa_gpu_store42() { + // end-to-end: dispatch test_store kernel, verify 42.0 in output + if !is_available() { return; } + let mut dev = match HsaDevice::open() { + Ok(d) => d, + Err(e) => { println!(" SKIP: {}", e); return; }, + }; + if dev.kernels.is_none() { println!(" SKIP: no kernels"); return; } + + let n = 8u32; + + // verify code object in VRAM + { + let co = &dev._code_objects[0]; // test_store + let code_va_offset = 0x1300usize; // .text VA in the image + let code_bytes = co.buf.read(code_va_offset, 16); + println!(" code at va_offset 0x1300: {}", code_bytes.iter().map(|b| format!("{:02x}", b)).collect::>().join(" ")); + let is_zero = code_bytes.iter().all(|&b| b == 0); + println!(" code is zero: {}", is_zero); + } + + // allocate output buffer in VRAM (gpu can write, cpu reads via BAR) + let y_buf = dev.alloc.alloc_vram(4096).unwrap(); + // write sentinel to verify it gets overwritten + let sentinel = vec![-1.0f32; n as usize]; + y_buf.write_f32(0, &sentinel); + + // kernargs: just y pointer (8 bytes) + let mut args = KernArgs::new(); + args.push_ptr(&y_buf); + // pad to kernarg_size (8 bytes for test_store) + let args_buf = args.upload(&dev.alloc).unwrap(); + + // dispatch + let ok = dev.dispatch_kernel("test_store", &args_buf, [n, 1, 1], [n, 1, 1]); + assert!(ok, "test_store dispatch timed out"); + + // read back via cpu_ptr (direct BAR read, no DMA) + let y = y_buf.read_f32(0, n as usize); + println!(" gpu output: {:?}", y); + for i in 0..n as usize { + assert!((y[i] - 42.0).abs() < 0.01, + "y[{}] = {} expected 42.0", i, y[i]); + } + println!(" test_store: CORRECT — gpu wrote 42.0 to {} elements", n); + } + + #[test] + #[ignore] + fn test_hsa_signal_only() { + if !is_available() { return; } + let mut dev = match HsaDevice::open() { + Ok(d) => d, + Err(e) => { println!(" OPEN FAILED: {}", e); return; }, + }; + + // Just send release_mem to write a signal — no compute dispatch + dev.signal.write(0, &[0u8; 4]); // zero the signal + dev.signal_value = 42; + + println!(" signal_addr: 0x{:x}", dev.signal.va_addr); + println!(" mailbox_ptr: 0x{:x}", dev.queue_event_mailbox_ptr); + println!(" event_id: {}", dev.queue_event_id); + println!(" ring_addr: 0x{:x}", dev.queue.ring.va_addr); + println!(" put before: {}", dev.queue.put); + + dev.queue.signal(dev.signal.va_addr, 42, + dev.queue_event_mailbox_ptr, dev.queue_event_id); + + println!(" put after signal: {}", dev.queue.put); + + // Dump ring contents before submit + let ring_u32 = dev.queue.ring.as_slice::(); + println!(" ring contents (first {} DWORDs):", dev.queue.put); + for i in 0..dev.queue.put as usize { + println!(" [{:3}] 0x{:08x}", i, ring_u32[i]); + } + + dev.queue.submit(); + + let signal_ptr = dev.signal.cpu_ptr as *const u32; + let ok = ComputeQueue::poll_signal(signal_ptr, 42, 2_000_000); + let val = unsafe { signal_ptr.read_volatile() }; + println!(" signal_only: ok={} val={}", ok, val); + assert!(ok, "signal-only dispatch timed out (val={})", val); + } + + #[test] + fn test_hsa_backend_matvec() { + if !is_available() { return; } + let backend = match HsaBackend::new() { + Ok(b) => b, + Err(_) => return, + }; + + // Small matvec → should go to CPU (AVX-512) + let weight = vec![1.0f32, 2.0, 3.0, 4.0]; // 2x2 + let bias = vec![0.5, 0.5]; + let x = vec![1.0, 1.0]; + let mut y = vec![0.0; 2]; + backend.matvec(&weight, &bias, &x, &mut y, 2, 2); + assert!((y[0] - 3.5).abs() < 1e-5); // 1*1 + 2*1 + 0.5 + assert!((y[1] - 7.5).abs() < 1e-5); // 3*1 + 4*1 + 0.5 + eprintln!(" HsaBackend matvec (CPU): OK"); + } + + #[test] + fn test_hsa_kernels_loaded() { + if !is_available() { return; } + let dev = match HsaDevice::open() { + Ok(d) => d, + Err(e) => { println!(" OPEN FAILED: {}", e); return; }, + }; + + let km = dev.kernels.as_ref().expect("kernels not loaded"); + assert!(km.contains_key("test_store"), "test_store kernel missing"); + assert!(km.contains_key("matvec"), "matvec kernel missing"); + + let mv = &km["matvec"]; + println!(" matvec: rsrc1=0x{:08x} rsrc2=0x{:08x} code=0x{:x} karg={}", + mv.desc.pgm_rsrc1, mv.desc.pgm_rsrc2, mv.code_addr, mv.desc.kernarg_size); + let ts = &km["test_store"]; + println!(" test_store: rsrc1=0x{:08x} code=0x{:x} karg={}", + ts.desc.pgm_rsrc1, ts.code_addr, ts.desc.kernarg_size); + println!(" {} kernels: {:?}", km.len(), km.keys().collect::>()); + } + + #[test] + fn test_hsa_dispatch_dryrun() { + // Dump PM4 packets WITHOUT submitting — safe, no GPU crash + if !is_available() { return; } + let mut dev = match HsaDevice::open() { + Ok(d) => d, + Err(e) => { println!(" OPEN FAILED: {}", e); return; }, + }; + if dev.kernels.is_none() { println!(" NO KERNELS"); return; } + + let out_dim = 64u32; + let in_dim = 64u32; + let weight = vec![0.0f32; (out_dim * in_dim) as usize]; + let bias = vec![0.0f32; out_dim as usize]; + let x = vec![0.0f32; in_dim as usize]; + + let w_buf = dev.upload_f32(&weight).unwrap(); + let b_buf = dev.upload_f32(&bias).unwrap(); + let x_buf = dev.upload_f32(&x).unwrap(); + let y_buf = dev.alloc_output(out_dim as usize * 4).unwrap(); + + let grid = [out_dim, 1, 1]; + let block = [64.min(out_dim), 1, 1]; + let kernarg_size = dev.kernels.as_ref().unwrap() + ["matvec"].desc.kernarg_size; + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); + args.push_ptr(&b_buf); + args.push_ptr(&x_buf); + args.push_ptr(&y_buf); + args.push_u32(out_dim); + args.push_u32(in_dim); + args.fill_hidden_args(grid, block, kernarg_size); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let entry = dev.kernels.as_ref().unwrap()["matvec"].clone(); + + // Build packets but DON'T submit + dev.queue.dispatch( + entry.code_addr, + entry.desc.pgm_rsrc1, + entry.desc.pgm_rsrc2, + entry.desc.pgm_rsrc3, + args_buf.va_addr, + dev.scratch.va_addr, + grid, block, + ); + + // Dump + let ring = dev.queue.ring.as_slice::(); + let n = dev.queue.put as usize; + println!(" Our PM4 dispatch ({} DWORDs):", n); + for i in 0..n { + println!(" [{:3}] 0x{:08x}", i, ring[i]); + } + println!(" code_addr: 0x{:x}", entry.code_addr); + println!(" kernargs: 0x{:x}", args_buf.va_addr); + println!(" rsrc1: 0x{:08x} rsrc2: 0x{:08x} rsrc3: 0x{:08x}", + entry.desc.pgm_rsrc1, entry.desc.pgm_rsrc2, entry.desc.pgm_rsrc3); + + // Don't submit — just verify packet structure + assert!(n > 0, "no packets generated"); + } + + #[test] + fn test_hsa_gpu_matvec_dispatch() { + if !is_available() { return; } + let mut dev = match HsaDevice::open() { + Ok(d) => d, + Err(e) => { println!(" SKIP: {}", e); return; }, + }; + if dev.kernels.is_none() { println!(" SKIP: no kernels"); return; } + + let out_dim = 4u32; + let in_dim = 4u32; + + // W = identity matrix + let mut weight = vec![0.0f32; (out_dim * in_dim) as usize]; + for i in 0..out_dim as usize { + weight[i * in_dim as usize + i] = 1.0; + } + let bias = vec![0.5f32; out_dim as usize]; + let x: Vec = (1..=in_dim).map(|i| i as f32).collect(); // [1,2,3,4] + + let w_buf = dev.upload_f32(&weight).unwrap(); + let b_buf = dev.upload_f32(&bias).unwrap(); + let x_buf = dev.upload_f32(&x).unwrap(); + let y_buf = dev.alloc.alloc_userptr_public(4096).unwrap(); + + // raw asm kernel: 40 bytes kernargs, no hidden args + let grid = [out_dim, 1, 1]; + let block = [out_dim, 1, 1]; + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); + args.push_ptr(&b_buf); + args.push_ptr(&x_buf); + args.push_ptr(&y_buf); + args.push_u32(out_dim); + args.push_u32(in_dim); + let args_buf = args.upload(&dev.alloc).unwrap(); + + // Dispatch + let ok = dev.dispatch_kernel("matvec", &args_buf, grid, block); + assert!(ok, "GPU matvec dispatch timed out"); + + // Read back and verify + let y = y_buf.read_f32(0, out_dim as usize); + println!(" gpu matvec result: {:?}", y); + // identity * [1,2,3,4] + 0.5 = [1.5, 2.5, 3.5, 4.5] + let expected: Vec = (1..=out_dim).map(|i| i as f32 + 0.5).collect(); + for i in 0..out_dim as usize { + assert!((y[i] - expected[i]).abs() < 1e-3, + "y[{}] = {} expected {}", i, y[i], expected[i]); + } + println!(" gpu matvec: CORRECT ({}x{})", out_dim, in_dim); + } + + #[test] + #[ignore] // DISABLED: PM4 packets crash GPU + fn test_hsa_gpu_matvec_large() { + if !is_available() { return; } + let mut dev = match HsaDevice::open() { + Ok(d) => d, + Err(_) => return, + }; + if dev.kernels.is_none() { return; } + + // CTM-sized: 1024x1024 matvec + let out_dim = 1024u32; + let in_dim = 1024u32; + let n = (out_dim * in_dim) as usize; + + // Random-ish weights + let weight: Vec = (0..n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect(); + let bias = vec![0.1f32; out_dim as usize]; + let x: Vec = (0..in_dim).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect(); + + // CPU reference + let mut y_cpu = vec![0.0f32; out_dim as usize]; + for row in 0..out_dim as usize { + let mut sum = bias[row]; + for col in 0..in_dim as usize { + sum += weight[row * in_dim as usize + col] * x[col]; + } + y_cpu[row] = sum; + } + + // GPU + let w_buf = dev.upload_f32(&weight).unwrap(); + let b_buf = dev.upload_f32(&bias).unwrap(); + let x_buf = dev.upload_f32(&x).unwrap(); + let y_buf = dev.alloc_output(out_dim as usize * 4).unwrap(); + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); + args.push_ptr(&b_buf); + args.push_ptr(&x_buf); + args.push_ptr(&y_buf); + args.push_u32(out_dim); + args.push_u32(in_dim); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let ok = dev.dispatch_kernel("matvec", &args_buf, + [out_dim, 1, 1], [256, 1, 1]); + assert!(ok, "GPU matvec dispatch timed out"); + + let y_gpu = y_buf.read_f32(0, out_dim as usize); + + // Verify + let mut max_err = 0.0f32; + for i in 0..out_dim as usize { + let err = (y_gpu[i] - y_cpu[i]).abs(); + max_err = max_err.max(err); + } + eprintln!(" GPU matvec 1024x1024: max_err={:.6}", max_err); + assert!(max_err < 0.1, "GPU matvec too inaccurate: max_err={}", max_err); + } + + #[test] + #[ignore] // DISABLED: PM4 packets crash GPU + fn test_hsa_performance_comparison() { + if !is_available() { return; } + let mut dev = match HsaDevice::open() { + Ok(d) => d, + Err(_) => return, + }; + if dev.kernels.is_none() { return; } + + use std::time::Instant; + + // Simple CPU matvec for comparison + fn cpu_matvec(w: &[f32], b: &[f32], x: &[f32], y: &mut [f32], out: usize, inp: usize) { + for r in 0..out { + let mut s = b[r]; + for c in 0..inp { s += w[r * inp + c] * x[c]; } + y[r] = s; + } + } + + eprintln!("\n ┌─────────────────────────────────────────────────┐"); + eprintln!(" │ HSA Performance: CPU (AVX-512) vs GPU (RDNA3) │"); + eprintln!(" ├──────────┬──────────┬──────────┬────────────────┤"); + eprintln!(" │ Size │ CPU (μs) │ GPU (μs) │ Winner │"); + eprintln!(" ├──────────┼──────────┼──────────┼────────────────┤"); + + for &(out_dim, in_dim) in &[ + (64, 64), (128, 128), (256, 256), (512, 512), + (1024, 1024), (2048, 2048), (4096, 1024), + ] { + let n = out_dim * in_dim; + let weight: Vec = (0..n).map(|i| (i % 100) as f32 / 100.0 - 0.5).collect(); + let bias = vec![0.1f32; out_dim]; + let x: Vec = (0..in_dim).map(|i| (i % 100) as f32 / 100.0).collect(); + let mut y = vec![0.0f32; out_dim]; + + // CPU benchmark (10 iterations, take median) + let mut cpu_times = Vec::new(); + for _ in 0..10 { + let t = Instant::now(); + cpu_matvec(&weight, &bias, &x, &mut y, out_dim, in_dim); + cpu_times.push(t.elapsed().as_micros() as u64); + } + cpu_times.sort(); + let cpu_us = cpu_times[5]; // median + + // GPU benchmark + let w_buf = dev.upload_f32(&weight).unwrap(); + let b_buf = dev.upload_f32(&bias).unwrap(); + let x_buf = dev.upload_f32(&x).unwrap(); + let y_buf = dev.alloc_output(out_dim * 4).unwrap(); + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); + args.push_ptr(&b_buf); + args.push_ptr(&x_buf); + args.push_ptr(&y_buf); + args.push_u32(out_dim as u32); + args.push_u32(in_dim as u32); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let block_size = 256.min(out_dim as u32); + + // Warmup + dev.dispatch_kernel("matvec", &args_buf, + [out_dim as u32, 1, 1], [block_size, 1, 1]); + + let mut gpu_times = Vec::new(); + for _ in 0..10 { + // Zero signal for fresh measurement + dev.signal.write(0, &(dev.signal_value + 1).to_le_bytes()[..4]); + let t = Instant::now(); + dev.dispatch_kernel("matvec", &args_buf, + [out_dim as u32, 1, 1], [block_size, 1, 1]); + gpu_times.push(t.elapsed().as_micros() as u64); + } + gpu_times.sort(); + let gpu_us = gpu_times[5]; // median + + let winner = if gpu_us < cpu_us { "GPU ←" } else { "CPU" }; + let flops = out_dim * in_dim * 2; + eprintln!(" │ {:4}x{:<4}│ {:>8} │ {:>8} │ {:>14} │", + out_dim, in_dim, cpu_us, gpu_us, winner); + } + eprintln!(" └──────────┴──────────┴──────────┴────────────────┘"); + } +} diff --git a/src/dispatch.rs b/src/dispatch.rs new file mode 100644 index 0000000..4b84696 --- /dev/null +++ b/src/dispatch.rs @@ -0,0 +1,413 @@ +//! Kernel dispatch: load code objects, set up kernel arguments, run. +//! +//! AMD code objects are ELF files containing multiple kernels. +//! Each kernel has a `.kd` descriptor in `.rodata` and ISA code in `.text`. +//! Symbol table maps kernel names to their descriptors. + +use crate::memory::{GpuAllocator, GpuBuffer}; +use crate::queue::ComputeQueue; +use std::collections::HashMap; + +/// Parsed kernel descriptor from an AMD code object. +#[derive(Debug, Clone)] +pub struct KernelDescriptor { + pub pgm_rsrc1: u32, + pub pgm_rsrc2: u32, + pub pgm_rsrc3: u32, + pub kernel_code_entry_offset: i64, + pub group_segment_size: u32, + pub private_segment_size: u32, + pub kernarg_size: u32, +} + +/// A single kernel within a loaded code object. +#[derive(Debug, Clone)] +pub struct KernelEntry { + pub name: String, + pub desc: KernelDescriptor, + pub code_addr: u64, +} + +/// A loaded code object in GPU memory, containing one or more kernels. +pub struct CodeObject { + pub buf: GpuBuffer, + pub kernels: HashMap, +} + +impl CodeObject { + /// Parse only (no GPU upload) — for diagnostics. + pub fn parse_only(elf_data: &[u8]) -> usize { + match parse_elf_kernels(elf_data, 0) { + Some(v) => v.len(), + None => 0, + } + } + + /// Load a code object ELF into GPU memory and parse all kernels. + /// + /// The ELF is loaded as a flat VA-indexed image: we allocate enough space + /// for the highest VA in the ELF, then copy each LOAD segment to its VA offset. + /// This way VA addresses in the code work correctly as offsets from gpu_base. + pub fn load(alloc: &GpuAllocator, elf_data: &[u8]) -> std::io::Result { + // Build VA-indexed image (like tinygrad's elf_loader) + let image = build_va_image(elf_data) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, + "failed to parse ELF"))?; + + let size = ((image.len() + 4095) & !4095) as u64; + // VRAM with PUBLIC — code lives in GPU memory, CPU-accessible via BAR for upload + let buf = alloc.alloc_vram(size)?; + buf.write(0, &image); + + let kernels = parse_elf_kernels(elf_data, buf.va_addr) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, + "failed to parse code object ELF"))?; + + let mut map = HashMap::new(); + for k in kernels { + map.insert(k.name.clone(), k); + } + + Ok(CodeObject { buf, kernels: map }) + } + + /// Get a kernel by name. + pub fn kernel(&self, name: &str) -> Option<&KernelEntry> { + self.kernels.get(name) + } +} + +/// A ready-to-dispatch GPU program (single kernel from a code object). +pub struct GpuProgram { + pub code_buf: GpuBuffer, + pub desc: KernelDescriptor, + pub code_addr: u64, +} + +impl GpuProgram { + /// Load a single-kernel code object. + pub fn load(alloc: &GpuAllocator, code_object: &[u8]) -> std::io::Result { + let co = CodeObject::load(alloc, code_object)?; + let first = co.kernels.values().next() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, + "no kernels in code object"))?; + Ok(GpuProgram { + desc: first.desc.clone(), + code_addr: first.code_addr, + code_buf: co.buf, + }) + } + + /// Dispatch and signal. + pub fn dispatch(&self, queue: &mut ComputeQueue, + kernargs: &GpuBuffer, scratch_addr: u64, + grid: [u32; 3], block: [u32; 3], + signal: &GpuBuffer, signal_value: u32, + event_mailbox_ptr: u64, event_id: u32) { + queue.dispatch_lds( + self.code_addr, + self.desc.pgm_rsrc1, + self.desc.pgm_rsrc2, + self.desc.pgm_rsrc3, + kernargs.va_addr, + scratch_addr, + grid, block, + self.desc.group_segment_size, + ); + queue.signal(signal.va_addr, signal_value, event_mailbox_ptr, event_id); + queue.submit(); + } +} + +// ─── ELF parsing ──────────────────────────────────────────── + +/// ELF64 symbol types +const STT_OBJECT: u8 = 1; +const STT_FUNC: u8 = 2; + +/// Parse all kernels from an ELF code object. +/// Each kernel has a `.kd` OBJECT symbol in .rodata (the descriptor) +/// and a `` FUNC symbol in .text (the code entry). +fn parse_elf_kernels(elf: &[u8], gpu_base: u64) -> Option> { + if elf.len() < 64 || &elf[0..4] != b"\x7fELF" { return None; } + + let e_shoff = r64(elf, 40) as usize; + let e_shentsize = r16(elf, 58) as usize; + let e_shnum = r16(elf, 60) as usize; + let e_shstrndx = r16(elf, 62) as usize; + + // Get section name string table + let shstrtab_sh = e_shoff + e_shstrndx * e_shentsize; + let shstrtab_off = r64(elf, shstrtab_sh + 24) as usize; + + // Find section headers by name + let mut symtab_off = 0usize; + let mut symtab_size = 0usize; + let mut symtab_entsize = 0usize; + let mut symtab_link = 0usize; + let mut rodata_off = 0usize; + let mut rodata_addr = 0u64; + + for i in 0..e_shnum { + let sh = e_shoff + i * e_shentsize; + if sh + e_shentsize > elf.len() { break; } + + let sh_name_idx = r32(elf, sh) as usize; + let sh_type = r32(elf, sh + 4); + let sh_addr = r64(elf, sh + 16); + let sh_offset = r64(elf, sh + 24) as usize; + let sh_size = r64(elf, sh + 32) as usize; + + let name = read_str(elf, shstrtab_off + sh_name_idx); + + if sh_type == 2 { // SHT_SYMTAB + symtab_off = sh_offset; + symtab_size = sh_size; + symtab_entsize = r64(elf, sh + 56) as usize; + symtab_link = r32(elf, sh + 40) as usize; + } + if name == ".rodata" { + rodata_off = sh_offset; + rodata_addr = sh_addr; + } + } + + if symtab_entsize == 0 { return None; } + + // Get strtab + let strtab_sh = e_shoff + symtab_link * e_shentsize; + let strtab_off = r64(elf, strtab_sh + 24) as usize; + + // Parse symbols: collect .kd descriptors and FUNC addresses + let mut kd_map: HashMap = HashMap::new(); // name -> (vaddr, file_offset) + let mut func_map: HashMap = HashMap::new(); // name -> vaddr + + let n_syms = symtab_size / symtab_entsize; + for i in 0..n_syms { + let sym = symtab_off + i * symtab_entsize; + if sym + symtab_entsize > elf.len() { break; } + + let st_name = r32(elf, sym) as usize; + let st_info = elf[sym + 4]; + let st_value = r64(elf, sym + 8); + let _st_size = r64(elf, sym + 16); + let st_type = st_info & 0xf; + + let name = read_str(elf, strtab_off + st_name); + + if st_type == STT_OBJECT && name.ends_with(".kd") { + let kernel_name = name[..name.len() - 3].to_string(); + // .kd symbol value is the VA in the ELF. Since we upload VA-indexed image, + // the file_off in the original ELF for reading the descriptor is rodata_off + (va - rodata_addr) + // but the GPU offset is just the VA itself (since image is VA-indexed). + let elf_file_off = rodata_off + (st_value - rodata_addr) as usize; + kd_map.insert(kernel_name, (st_value, elf_file_off)); + } else if st_type == STT_FUNC && !name.starts_with("__clang") { + func_map.insert(name.to_string(), st_value); + } + } + + // Match descriptors with code addresses + let mut kernels = Vec::new(); + for (name, (kd_vaddr, kd_file_off)) in &kd_map { + if kd_file_off + 64 > elf.len() { continue; } + let kd = &elf[*kd_file_off..]; + + // AMDHSA kernel descriptor layout (verified against tinygrad): + // +00: group_segment_fixed_size (u32) + // +04: private_segment_fixed_size (u32) + // +08: kernarg_size (u32) + // +0c: reserved (u32) + // +10: kernel_code_entry_byte_offset (i64) + // +18: reserved (20 bytes) + // +2c: compute_pgm_rsrc3 (u32) + // +30: compute_pgm_rsrc1 (u32) + // +34: compute_pgm_rsrc2 (u32) + // +38: kernel_code_properties (u16) + kernarg_preload (u16) + let desc = KernelDescriptor { + group_segment_size: u32::from_le_bytes(kd[0x00..0x04].try_into().unwrap()), + private_segment_size: u32::from_le_bytes(kd[0x04..0x08].try_into().unwrap()), + kernarg_size: u32::from_le_bytes(kd[0x08..0x0c].try_into().unwrap()), + kernel_code_entry_offset: i64::from_le_bytes(kd[0x10..0x18].try_into().unwrap()), + pgm_rsrc3: u32::from_le_bytes(kd[0x2c..0x30].try_into().unwrap()), + pgm_rsrc1: u32::from_le_bytes(kd[0x30..0x34].try_into().unwrap()), + pgm_rsrc2: u32::from_le_bytes(kd[0x34..0x38].try_into().unwrap()), + }; + + // Code address: since we upload VA-indexed image, gpu_base + kd_vaddr = descriptor addr. + // kernel_code_entry_offset is relative to the descriptor's VA. + let kd_gpu_addr = gpu_base + *kd_vaddr; + let code_addr = (kd_gpu_addr as i64 + desc.kernel_code_entry_offset) as u64; + + eprintln!(" kernel '{}': kd_va=0x{:x} file_off=0x{:x} entry_off=0x{:x} code=0x{:x}", + name, kd_vaddr, kd_file_off, desc.kernel_code_entry_offset, code_addr); + + kernels.push(KernelEntry { + name: name.clone(), + desc, + code_addr, + }); + } + + Some(kernels) +} + +/// Build a VA-indexed image from an ELF: allocate buffer sized to max VA, +/// copy each LOAD segment to its VA offset. This way gpu_base + VA = correct GPU address. +fn build_va_image(elf: &[u8]) -> Option> { + if elf.len() < 64 || &elf[0..4] != b"\x7fELF" { return None; } + + let e_phoff = r64(elf, 32) as usize; + let e_phentsize = r16(elf, 54) as usize; + let e_phnum = r16(elf, 56) as usize; + + // Find max VA across all LOAD segments + let mut max_va: usize = 0; + for i in 0..e_phnum { + let ph = e_phoff + i * e_phentsize; + if ph + e_phentsize > elf.len() { break; } + let p_type = r32(elf, ph); + if p_type != 1 { continue; } // PT_LOAD = 1 + let p_vaddr = r64(elf, ph + 16) as usize; + let p_memsz = r64(elf, ph + 40) as usize; + max_va = max_va.max(p_vaddr + p_memsz); + } + + if max_va == 0 { return None; } + let mut image = vec![0u8; max_va]; + + // Copy each LOAD segment to its VA position + for i in 0..e_phnum { + let ph = e_phoff + i * e_phentsize; + if ph + e_phentsize > elf.len() { break; } + let p_type = r32(elf, ph); + if p_type != 1 { continue; } + let p_offset = r64(elf, ph + 8) as usize; + let p_vaddr = r64(elf, ph + 16) as usize; + let p_filesz = r64(elf, ph + 32) as usize; + let end = (p_offset + p_filesz).min(elf.len()); + let copy_len = end - p_offset; + if p_vaddr + copy_len <= image.len() { + image[p_vaddr..p_vaddr + copy_len].copy_from_slice(&elf[p_offset..p_offset + copy_len]); + } + } + + Some(image) +} + +fn r16(data: &[u8], off: usize) -> u16 { + u16::from_le_bytes(data[off..off+2].try_into().unwrap()) +} +fn r32(data: &[u8], off: usize) -> u32 { + u32::from_le_bytes(data[off..off+4].try_into().unwrap()) +} +fn r64(data: &[u8], off: usize) -> u64 { + u64::from_le_bytes(data[off..off+8].try_into().unwrap()) +} + +fn read_str(data: &[u8], off: usize) -> &str { + let end = data[off..].iter().position(|&b| b == 0).unwrap_or(0); + std::str::from_utf8(&data[off..off + end]).unwrap_or("") +} + +// ─── Kernel argument builder ──────────────────────────────── + +/// Helper to pack kernel arguments into a GPU buffer. +pub struct KernArgs { + data: Vec, +} + +impl KernArgs { + pub fn new() -> Self { + KernArgs { data: Vec::with_capacity(256) } + } + + pub fn push_ptr(&mut self, buf: &GpuBuffer) { + self.align(8); + self.data.extend_from_slice(&buf.va_addr.to_le_bytes()); + } + + pub fn push_u64(&mut self, val: u64) { + self.align(8); + self.data.extend_from_slice(&val.to_le_bytes()); + } + + pub fn push_u32(&mut self, val: u32) { + self.align(4); + self.data.extend_from_slice(&val.to_le_bytes()); + } + + pub fn push_f32(&mut self, val: f32) { + self.align(4); + self.data.extend_from_slice(&val.to_le_bytes()); + } + + fn align(&mut self, alignment: usize) { + let pad = (alignment - (self.data.len() % alignment)) % alignment; + self.data.extend(std::iter::repeat(0u8).take(pad)); + } + + /// Fill OpenCL hidden arguments based on dispatch grid/block sizes. + /// Must be called after all explicit args are pushed but before upload. + /// Standard hidden arg layout (from clang OpenCL compiler): + /// +0x00 from end of explicit: hidden_block_count_x/y/z (3x u32) + /// +0x0c: hidden_group_size_x/y/z (3x u16) + /// +0x12: hidden_remainder_x/y/z (3x u16) + /// +0x18: (padding to 0x28 from end of explicit) + /// +0x28: hidden_global_offset_x/y/z (3x u64) + /// +0x40: hidden_grid_dims (u16) + pub fn fill_hidden_args(&mut self, grid: [u32; 3], block: [u32; 3], kernarg_size: u32) { + // Pad to the expected kernarg_size + while self.data.len() < kernarg_size as usize { + self.data.push(0); + } + + // Block counts = grid / block (number of workgroups per dimension) + let block_count = [ + if block[0] > 0 { grid[0] / block[0] } else { 0 }, + if block[1] > 0 { grid[1] / block[1] } else { 0 }, + if block[2] > 0 { grid[2] / block[2] } else { 0 }, + ]; + let remainder = [ + if block[0] > 0 { (grid[0] % block[0]) as u16 } else { 0 }, + if block[1] > 0 { (grid[1] % block[1]) as u16 } else { 0 }, + if block[2] > 0 { (grid[2] % block[2]) as u16 } else { 0 }, + ]; + let grid_dims: u16 = if grid[2] > 1 { 3 } else if grid[1] > 1 { 2 } else { 1 }; + + // Write at standard offsets (relative to byte 0x28 in our matvec case) + // These offsets come from the .note metadata: hidden_block_count_x at 0x28 + let base = 0x28usize; // first hidden arg offset for our kernels + if base + 12 <= self.data.len() { + self.data[base..base+4].copy_from_slice(&block_count[0].to_le_bytes()); + self.data[base+4..base+8].copy_from_slice(&block_count[1].to_le_bytes()); + self.data[base+8..base+12].copy_from_slice(&block_count[2].to_le_bytes()); + } + let gs = base + 12; // hidden_group_size + if gs + 6 <= self.data.len() { + self.data[gs..gs+2].copy_from_slice(&(block[0] as u16).to_le_bytes()); + self.data[gs+2..gs+4].copy_from_slice(&(block[1] as u16).to_le_bytes()); + self.data[gs+4..gs+6].copy_from_slice(&(block[2] as u16).to_le_bytes()); + } + let rm = gs + 6; // hidden_remainder + if rm + 6 <= self.data.len() { + self.data[rm..rm+2].copy_from_slice(&remainder[0].to_le_bytes()); + self.data[rm+2..rm+4].copy_from_slice(&remainder[1].to_le_bytes()); + self.data[rm+4..rm+6].copy_from_slice(&remainder[2].to_le_bytes()); + } + // hidden_global_offset at 0x50 + // (all zeros — we don't use global offsets) + // hidden_grid_dims at 0x68 + if 0x68 + 2 <= self.data.len() { + self.data[0x68..0x6a].copy_from_slice(&grid_dims.to_le_bytes()); + } + } + + pub fn upload(&self, alloc: &GpuAllocator) -> std::io::Result { + let size = ((self.data.len().max(256) + 4095) & !4095) as u64; + // Kernargs must be GPU-readable (PUBLIC flag) + let buf = alloc.alloc_userptr_public(size)?; + buf.write(0, &self.data); + Ok(buf) + } +} diff --git a/src/ioctl.rs b/src/ioctl.rs new file mode 100644 index 0000000..cc9d4a8 --- /dev/null +++ b/src/ioctl.rs @@ -0,0 +1,330 @@ +//! Raw KFD ioctl bindings for AMD GPU access. +//! +//! Structs match /usr/include/linux/kfd_ioctl.h exactly. +//! No dependencies beyond libc. + +use std::os::unix::io::RawFd; + +// ─── ioctl direction bits ─────────────────────────────────── + +const IOC_NONE: u32 = 0; +const IOC_WRITE: u32 = 1; +const IOC_READ: u32 = 2; +const IOC_READWRITE: u32 = 3; + +const AMDKFD_IOCTL_BASE: u32 = b'K' as u32; + +const fn ioc(dir: u32, nr: u32, size: u32) -> u64 { + ((dir as u64) << 30) | ((size as u64) << 16) | ((AMDKFD_IOCTL_BASE as u64) << 8) | (nr as u64) +} + +const fn ior(nr: u32) -> u64 { ioc(IOC_READ, nr, std::mem::size_of::() as u32) } +const fn iow(nr: u32) -> u64 { ioc(IOC_WRITE, nr, std::mem::size_of::() as u32) } +const fn iowr(nr: u32) -> u64 { ioc(IOC_READWRITE, nr, std::mem::size_of::() as u32) } + +unsafe fn kfd_ioctl(fd: RawFd, request: u64, arg: &mut T) -> std::io::Result<()> { + let ret = libc::ioctl(fd, request as libc::c_ulong, arg as *mut T); + if ret < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +// ─── Memory allocation flags ──────────────────────────────── + +pub const ALLOC_MEM_FLAGS_VRAM: u32 = 1 << 0; +pub const ALLOC_MEM_FLAGS_GTT: u32 = 1 << 1; +pub const ALLOC_MEM_FLAGS_USERPTR: u32 = 1 << 2; +pub const ALLOC_MEM_FLAGS_DOORBELL: u32 = 1 << 3; +pub const ALLOC_MEM_FLAGS_MMIO_REMAP: u32 = 1 << 4; +pub const ALLOC_MEM_FLAGS_WRITABLE: u32 = 1 << 31; +pub const ALLOC_MEM_FLAGS_EXECUTABLE: u32 = 1 << 30; +pub const ALLOC_MEM_FLAGS_PUBLIC: u32 = 1 << 29; +pub const ALLOC_MEM_FLAGS_NO_SUBSTITUTE: u32 = 1 << 28; +pub const ALLOC_MEM_FLAGS_AQL_QUEUE_MEM: u32 = 1 << 27; +pub const ALLOC_MEM_FLAGS_COHERENT: u32 = 1 << 26; +pub const ALLOC_MEM_FLAGS_UNCACHED: u32 = 1 << 25; + +// ─── Queue types ──────────────────────────────────────────── + +pub const QUEUE_TYPE_COMPUTE: u32 = 0; +pub const QUEUE_TYPE_SDMA: u32 = 1; +pub const QUEUE_TYPE_COMPUTE_AQL: u32 = 2; + +pub const MAX_QUEUE_PERCENTAGE: u32 = 100; +pub const MAX_QUEUE_PRIORITY: u32 = 15; + +// ─── Event types ──────────────────────────────────────────── + +pub const EVENT_SIGNAL: u32 = 0; +pub const EVENT_HW_EXCEPTION: u32 = 3; +pub const EVENT_MEMORY: u32 = 8; + +// ─── Doorbell mmap ────────────────────────────────────────── + +pub const MMAP_TYPE_DOORBELL: u64 = 0x3 << 62; + +// ─── Structs ──────────────────────────────────────────────── + +#[repr(C)] +#[derive(Debug, Default)] +pub struct GetVersionArgs { + pub major_version: u32, + pub minor_version: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct AcquireVmArgs { + pub drm_fd: u32, + pub gpu_id: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct CreateQueueArgs { + pub ring_base_address: u64, + pub write_pointer_address: u64, + pub read_pointer_address: u64, + pub doorbell_offset: u64, + pub ring_size: u32, + pub gpu_id: u32, + pub queue_type: u32, + pub queue_percentage: u32, + pub queue_priority: u32, + pub queue_id: u32, + pub eop_buffer_address: u64, + pub eop_buffer_size: u64, + pub ctx_save_restore_address: u64, + pub ctx_save_restore_size: u32, + pub ctl_stack_size: u32, + pub sdma_engine_id: u32, + pub pad: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct DestroyQueueArgs { + pub queue_id: u32, + pub pad: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct AllocMemoryArgs { + pub va_addr: u64, + pub size: u64, + pub handle: u64, + pub mmap_offset: u64, + pub gpu_id: u32, + pub flags: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct FreeMemoryArgs { + pub handle: u64, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct MapMemoryArgs { + pub handle: u64, + pub device_ids_array_ptr: u64, + pub n_devices: u32, + pub n_success: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct UnmapMemoryArgs { + pub handle: u64, + pub device_ids_array_ptr: u64, + pub n_devices: u32, + pub n_success: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct CreateEventArgs { + pub event_page_offset: u64, + pub event_trigger_data: u32, + pub event_type: u32, + pub auto_reset: u32, + pub node_id: u32, + pub event_id: u32, + pub event_slot_index: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct DestroyEventArgs { + pub event_id: u32, + pub pad: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct SetEventArgs { + pub event_id: u32, + pub pad: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct WaitEventsArgs { + pub events_ptr: u64, + pub num_events: u32, + pub wait_for_all: u32, + pub timeout: u32, + pub wait_result: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct RuntimeEnableArgs { + pub r_debug: u64, + pub mode_mask: u32, + pub capabilities_mask: u32, +} + +#[repr(C)] +#[derive(Debug, Default)] +pub struct GetAvailableMemoryArgs { + pub available: u64, + pub gpu_id: u32, + pub pad: u32, +} + +// kfd_event_data — used in wait_events array +// Union of memory_exception / hw_exception / signal_event, followed by ext + event_id + pad +// We only need the signal case, which is simplest +#[repr(C)] +#[derive(Debug, Default, Clone, Copy)] +pub struct EventData { + // Union — signal_event_data is just u64 (last_event_age) + // memory_exception_data is the largest at 32 bytes + // We use raw bytes for the union + pub union_data: [u8; 32], + pub kfd_event_data_ext: u64, + pub event_id: u32, + pub pad: u32, +} + +// ─── ioctl numbers ────────────────────────────────────────── + +const IOC_GET_VERSION: u64 = ior::(0x01); +const IOC_CREATE_QUEUE: u64 = iowr::(0x02); +const IOC_DESTROY_QUEUE: u64 = iowr::(0x03); +const IOC_CREATE_EVENT: u64 = iowr::(0x08); +const IOC_DESTROY_EVENT: u64 = iow::(0x09); +const IOC_SET_EVENT: u64 = iow::(0x0A); +const IOC_WAIT_EVENTS: u64 = iowr::(0x0C); +const IOC_ACQUIRE_VM: u64 = iow::(0x15); +const IOC_ALLOC_MEMORY: u64 = iowr::(0x16); +const IOC_FREE_MEMORY: u64 = iow::(0x17); +const IOC_MAP_MEMORY: u64 = iowr::(0x18); +const IOC_UNMAP_MEMORY: u64 = iowr::(0x19); +const IOC_AVAILABLE_MEMORY: u64 = iowr::(0x23); +const IOC_RUNTIME_ENABLE: u64 = iowr::(0x25); + +// ─── Typed wrappers ───────────────────────────────────────── + +pub fn get_version(fd: RawFd) -> std::io::Result<(u32, u32)> { + let mut args = GetVersionArgs::default(); + unsafe { kfd_ioctl(fd, IOC_GET_VERSION, &mut args)?; } + Ok((args.major_version, args.minor_version)) +} + +pub fn acquire_vm(fd: RawFd, drm_fd: RawFd, gpu_id: u32) -> std::io::Result<()> { + let mut args = AcquireVmArgs { drm_fd: drm_fd as u32, gpu_id }; + unsafe { kfd_ioctl(fd, IOC_ACQUIRE_VM, &mut args) } +} + +pub fn runtime_enable(fd: RawFd) -> std::io::Result { + let mut args = RuntimeEnableArgs::default(); + unsafe { kfd_ioctl(fd, IOC_RUNTIME_ENABLE, &mut args)?; } + #[cfg(debug_assertions)] + eprintln!(" runtime_enable: caps=0x{:x}", args.capabilities_mask); + Ok(args.capabilities_mask) +} + +pub fn alloc_memory(fd: RawFd, va_addr: u64, size: u64, gpu_id: u32, + flags: u32, mmap_offset: u64) -> std::io::Result { + let mut args = AllocMemoryArgs { + va_addr, size, gpu_id, flags, mmap_offset, handle: 0, + }; + unsafe { kfd_ioctl(fd, IOC_ALLOC_MEMORY, &mut args)?; } + Ok(args) +} + +pub fn free_memory(fd: RawFd, handle: u64) -> std::io::Result<()> { + let mut args = FreeMemoryArgs { handle }; + unsafe { kfd_ioctl(fd, IOC_FREE_MEMORY, &mut args) } +} + +pub fn map_memory(fd: RawFd, handle: u64, gpu_ids: &[u32]) -> std::io::Result { + let mut args = MapMemoryArgs { + handle, + device_ids_array_ptr: gpu_ids.as_ptr() as u64, + n_devices: gpu_ids.len() as u32, + n_success: 0, + }; + unsafe { kfd_ioctl(fd, IOC_MAP_MEMORY, &mut args)?; } + Ok(args.n_success) +} + +pub fn unmap_memory(fd: RawFd, handle: u64, gpu_ids: &[u32]) -> std::io::Result<()> { + let mut args = UnmapMemoryArgs { + handle, + device_ids_array_ptr: gpu_ids.as_ptr() as u64, + n_devices: gpu_ids.len() as u32, + n_success: 0, + }; + unsafe { kfd_ioctl(fd, IOC_UNMAP_MEMORY, &mut args)?; } + Ok(()) +} + +pub fn create_queue(fd: RawFd, args: &mut CreateQueueArgs) -> std::io::Result<()> { + unsafe { kfd_ioctl(fd, IOC_CREATE_QUEUE, args) } +} + +pub fn destroy_queue(fd: RawFd, queue_id: u32) -> std::io::Result<()> { + let mut args = DestroyQueueArgs { queue_id, pad: 0 }; + unsafe { kfd_ioctl(fd, IOC_DESTROY_QUEUE, &mut args) } +} + +pub fn create_event(fd: RawFd, event_type: u32, auto_reset: u32, + event_page_offset: u64) -> std::io::Result { + let mut args = CreateEventArgs { + event_page_offset, event_type, auto_reset, + ..Default::default() + }; + unsafe { kfd_ioctl(fd, IOC_CREATE_EVENT, &mut args)?; } + Ok(args) +} + +pub fn set_event(fd: RawFd, event_id: u32) -> std::io::Result<()> { + let mut args = SetEventArgs { event_id, pad: 0 }; + unsafe { kfd_ioctl(fd, IOC_SET_EVENT, &mut args) } +} + +pub fn wait_events(fd: RawFd, events: &mut [EventData], + wait_for_all: bool, timeout_ms: u32) -> std::io::Result { + let mut args = WaitEventsArgs { + events_ptr: events.as_mut_ptr() as u64, + num_events: events.len() as u32, + wait_for_all: wait_for_all as u32, + timeout: timeout_ms, + wait_result: 0, + }; + unsafe { kfd_ioctl(fd, IOC_WAIT_EVENTS, &mut args)?; } + Ok(args.wait_result) +} + +pub fn available_memory(fd: RawFd, gpu_id: u32) -> std::io::Result { + let mut args = GetAvailableMemoryArgs { gpu_id, available: 0, pad: 0 }; + unsafe { kfd_ioctl(fd, IOC_AVAILABLE_MEMORY, &mut args)?; } + Ok(args.available) +} diff --git a/src/kernels/addr_dump.co b/src/kernels/addr_dump.co new file mode 100755 index 0000000..e00a79e Binary files /dev/null and b/src/kernels/addr_dump.co differ diff --git a/src/kernels/addr_dump.s b/src/kernels/addr_dump.s new file mode 100644 index 0000000..49e5867 --- /dev/null +++ b/src/kernels/addr_dump.s @@ -0,0 +1,149 @@ +// Debug: dump computed addresses to Y buffer +// Y[lid*8 + 0] = v9 (W offset) +// Y[lid*8 + 1] = v11 (X offset) +// Y[lid*8 + 2] = s2 (W ptr lo) +// Y[lid*8 + 3] = s3 (W ptr hi) +// Y[lid*8 + 4] = s6 (X ptr lo) +// Y[lid*8 + 5] = s7 (X ptr hi) +// Y[lid*8 + 6] = s11 (K) +// Y[lid*8 + 7] = exec_lo + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.text +.globl addr_dump +.p2align 8 +.type addr_dump, @function +addr_dump: + s_mov_b32 s20, s2 + + s_load_b64 s[2:3], s[0:1], 0x00 + s_load_b64 s[4:5], s[0:1], 0x08 + s_load_b64 s[6:7], s[0:1], 0x10 + s_load_b64 s[8:9], s[0:1], 0x18 + s_load_b64 s[10:11], s[0:1], 0x20 + s_load_b32 s12, s[0:1], 0x28 + s_waitcnt lgkmcnt(0) + + s_add_u32 s13, s10, 31 + s_lshr_b32 s13, s13, 5 + s_mov_b32 s14, 0 + s_mov_b32 s15, s20 +.Ldiv: + s_cmp_lt_u32 s15, s13 + s_cbranch_scc1 .Ldiv_done + s_sub_u32 s15, s15, s13 + s_add_u32 s14, s14, 1 + s_branch .Ldiv +.Ldiv_done: + + v_and_b32 v1, 31, v0 + v_lshrrev_b32 v2, 5, v0 + s_lshl_b32 s16, s15, 5 + s_lshl_b32 s17, s14, 3 + v_add_nc_u32 v3, s16, v1 + v_add_nc_u32 v4, s17, v2 + + // bias load (same as matmul) + v_lshlrev_b32 v5, 2, v3 + v_mov_b32 v6, 0 + v_cmp_lt_u32 vcc_lo, v3, s10 + s_and_saveexec_b32 s18, vcc_lo + global_load_b32 v6, v5, s[4:5] + s_mov_b32 exec_lo, s18 + s_waitcnt vmcnt(0) + + // precompute (same as matmul) + v_lshrrev_b32 v7, 3, v0 + v_and_b32 v8, 7, v0 + v_lshlrev_b32 v8, 2, v8 + v_add_nc_u32 v9, s16, v7 + v_mul_lo_u32 v9, v9, s11 + v_add_nc_u32 v9, v9, v8 + v_lshlrev_b32 v9, 2, v9 + + v_lshlrev_b32 v10, 4, v0 + + v_add_nc_u32 v11, s17, v2 + v_mul_lo_u32 v11, v11, s11 + v_add_nc_u32 v11, v11, v1 + v_lshlrev_b32 v11, 2, v11 + + // dump 16 values per thread: Y[lid*16..lid*16+15] + v_lshlrev_b32 v20, 6, v0 // lid * 64 (16 dwords * 4 bytes) + + global_store_b32 v20, v9, s[8:9] offset:0 // [0] W offset + global_store_b32 v20, v11, s[8:9] offset:4 // [1] X offset + v_mov_b32 v21, s14 + global_store_b32 v20, v21, s[8:9] offset:8 // [2] s14 (wg_n) + v_mov_b32 v21, s15 + global_store_b32 v20, v21, s[8:9] offset:12 // [3] s15 (wg_m) + v_mov_b32 v21, s16 + global_store_b32 v20, v21, s[8:9] offset:16 // [4] s16 (wg_m*32) + v_mov_b32 v21, s17 + global_store_b32 v20, v21, s[8:9] offset:20 // [5] s17 (wg_n*8) + v_mov_b32 v21, s13 + global_store_b32 v20, v21, s[8:9] offset:24 // [6] s13 (num_wg_m) + v_mov_b32 v21, s20 + global_store_b32 v20, v21, s[8:9] offset:28 // [7] s20 (wg_id) + global_store_b32 v20, v1, s[8:9] offset:32 // [8] v1 (thread_m) + global_store_b32 v20, v2, s[8:9] offset:36 // [9] v2 (thread_n) + v_mov_b32 v21, s11 + global_store_b32 v20, v21, s[8:9] offset:40 // [10] s11 (K) + v_mov_b32 v21, s10 + global_store_b32 v20, v21, s[8:9] offset:44 // [11] s10 (M) + v_mov_b32 v21, s12 + global_store_b32 v20, v21, s[8:9] offset:48 // [12] s12 (N) + // intermediate: v11 before lshlrev = (s17+v2)*s11 + v1 + // let's compute it fresh + v_add_nc_u32 v21, s17, v2 + global_store_b32 v20, v21, s[8:9] offset:52 // [13] s17+v2 + v_mul_lo_u32 v21, v21, s11 + global_store_b32 v20, v21, s[8:9] offset:56 // [14] (s17+v2)*K + v_add_nc_u32 v21, v21, v1 + global_store_b32 v20, v21, s[8:9] offset:60 // [15] (s17+v2)*K + v1 + s_waitcnt vmcnt(0) + s_endpgm + +.rodata +.p2align 6 +.amdhsa_kernel addr_dump + .amdhsa_group_segment_fixed_size 0 + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 48 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 26 + .amdhsa_next_free_sgpr 21 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: addr_dump + .symbol: addr_dump.kd + .kernarg_segment_size: 48 + .group_segment_fixed_size: 0 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 21 + .vgpr_count: 26 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 16, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 24, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 32, .value_kind: by_value } + - { .size: 4, .offset: 36, .value_kind: by_value } + - { .size: 4, .offset: 40, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/coop_test.co b/src/kernels/coop_test.co new file mode 100755 index 0000000..5823f7b Binary files /dev/null and b/src/kernels/coop_test.co differ diff --git a/src/kernels/coop_test.s b/src/kernels/coop_test.s new file mode 100644 index 0000000..4379ab3 --- /dev/null +++ b/src/kernels/coop_test.s @@ -0,0 +1,92 @@ +// test cooperative W tile load to LDS +// each thread: load 4 floats from W, store to LDS, barrier, read own row back +// kernargs: W(u64) Y(u64) M(u32) K(u32) +// dispatch: 1 WG of 256 threads, computes for first 32x8 tile + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.text +.globl coop_test +.p2align 8 +.type coop_test, @function +coop_test: + s_load_b64 s[2:3], s[0:1], 0x00 // W + s_load_b64 s[4:5], s[0:1], 0x08 // Y + s_load_b64 s[6:7], s[0:1], 0x10 // M, K (s6=M, s7=K) + s_waitcnt lgkmcnt(0) + + // v0 = lid + v_and_b32 v1, 31, v0 // thread_m = lid & 31 + v_lshrrev_b32 v2, 5, v0 // thread_n = lid >> 5 + + // W coop load: tile_row = lid/8, tile_col = (lid%8)*4 + v_lshrrev_b32 v3, 3, v0 // tile_row + v_and_b32 v4, 7, v0 // lid & 7 + v_lshlrev_b32 v4, 2, v4 // tile_col = (lid&7)*4 + + // W global offset = (tile_row * K + tile_col) * 4 + v_mul_lo_u32 v5, v3, s7 // tile_row * K + v_add_nc_u32 v5, v5, v4 // + tile_col + v_lshlrev_b32 v5, 2, v5 // * 4 bytes + + // global load 4 floats + global_load_b128 v[6:9], v5, s[2:3] + s_waitcnt vmcnt(0) + + // LDS store offset = lid * 16 + v_lshlrev_b32 v10, 4, v0 + ds_store_b128 v10, v[6:9] + s_waitcnt lgkmcnt(0) + s_barrier + + // Now read back: thread (thread_m, thread_n) reads W[thread_m][0] from LDS + // LDS offset = thread_m * 32 * 4 = thread_m * 128 + v_lshlrev_b32 v11, 7, v1 // thread_m * 128 + ds_load_b32 v12, v11 // W[thread_m][0] + s_waitcnt lgkmcnt(0) + + // Store to Y[lid] = v12 + v_lshlrev_b32 v13, 2, v0 // lid * 4 + global_store_b32 v13, v12, s[4:5] + s_waitcnt vmcnt(0) + s_endpgm + +.rodata +.p2align 6 +.amdhsa_kernel coop_test + .amdhsa_group_segment_fixed_size 5120 + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 24 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 16 + .amdhsa_next_free_sgpr 8 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: coop_test + .symbol: coop_test.kd + .kernarg_segment_size: 24 + .group_segment_fixed_size: 5120 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 8 + .vgpr_count: 16 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 16, .value_kind: by_value } + - { .size: 4, .offset: 20, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/lds_test.co b/src/kernels/lds_test.co new file mode 100755 index 0000000..4fdcade Binary files /dev/null and b/src/kernels/lds_test.co differ diff --git a/src/kernels/lds_test.s b/src/kernels/lds_test.s new file mode 100644 index 0000000..337d7ac --- /dev/null +++ b/src/kernels/lds_test.s @@ -0,0 +1,72 @@ +// minimal LDS test: each thread stores lid to LDS, reads back neighbor's value +// output[lid] = lid + 1 (wrapped) to verify LDS sharing works +// kernarg: Y(u64) +// dispatch: global=256, local=256 + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.text +.globl lds_test +.p2align 8 +.type lds_test, @function +lds_test: + // s[0:1] = kernarg, s2 = wg_id (unused) + s_load_b64 s[2:3], s[0:1], 0x00 // Y ptr + s_waitcnt lgkmcnt(0) + + // v0 = lid + // store lid to LDS[lid*4] + v_lshlrev_b32 v1, 2, v0 // lid * 4 + v_mov_b32 v2, v0 // value = lid + ds_store_b32 v1, v2 + s_waitcnt lgkmcnt(0) + s_barrier + + // read neighbor: LDS[(lid+1)%256 * 4] + v_add_nc_u32 v3, v0, 1 + v_and_b32 v3, 255, v3 // (lid+1) % 256 + v_lshlrev_b32 v3, 2, v3 + ds_load_b32 v4, v3 + s_waitcnt lgkmcnt(0) + + // store to Y[lid] + global_store_b32 v1, v4, s[2:3] + s_waitcnt vmcnt(0) + s_endpgm + +.rodata +.p2align 6 +.amdhsa_kernel lds_test + .amdhsa_group_segment_fixed_size 1024 + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 8 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 8 + .amdhsa_next_free_sgpr 4 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: lds_test + .symbol: lds_test.kd + .kernarg_segment_size: 8 + .group_segment_fixed_size: 1024 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 4 + .vgpr_count: 8 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } +... +.end_amdgpu_metadata diff --git a/src/kernels/matmul.co b/src/kernels/matmul.co new file mode 100755 index 0000000..763ed47 Binary files /dev/null and b/src/kernels/matmul.co differ diff --git a/src/kernels/matmul.s b/src/kernels/matmul.s new file mode 100644 index 0000000..c1808f0 --- /dev/null +++ b/src/kernels/matmul.s @@ -0,0 +1,279 @@ +// rdna3 LDS-tiled matmul: Y = X * W^T + B +// +// Each workgroup computes a 32x8 tile of Y. +// 256 threads/WG: thread_m = lid%32, thread_n = lid/32 +// Tiles K in chunks of TK=32. Each tile iteration: +// 1. Cooperative load W[32][32] and X[8][32] to LDS +// 2. barrier +// 3. Each thread reads its W row and X row from LDS, does 32 FMAs +// 4. barrier +// +// LDS layout: W at 0 (4096B), X at 4096 (1024B), total 5120B +// +// Cooperative load assignment (W: 1024 elts / 256 threads = 4 each): +// tile_row = lid/8, tile_col = (lid%8)*4 +// global_load_b128 loads W[tile_row][tile_col..+3] +// ds_store_b128 at LDS offset lid*16 +// +// Cooperative load assignment (X: 256 elts / 256 threads = 1 each): +// x_row = lid/32, x_col = lid%32 +// global_load_b32 loads X[x_row][x_col] +// ds_store_b32 at LDS offset 4096 + lid*4 +// +// SGPR: s[0:1]=kernarg, s2=TGID_X +// kernargs (48B): W(u64) B(u64) X(u64) Y(u64) M(u32) K(u32) N(u32) +// dispatch: grid.x = ceil(M/32)*ceil(N/8)*256, block.x = 256 + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.set TM, 32 +.set TN, 8 +.set TK, 32 +.set LDS_X, 4096 // W uses 0..4095, X uses 4096..5119 +.set LDS_SZ, 5120 + +.text +.globl matmul +.p2align 8 +.type matmul, @function +matmul: + s_mov_b32 s20, s2 // save wg_id + + s_load_b64 s[2:3], s[0:1], 0x00 // W + s_load_b64 s[4:5], s[0:1], 0x08 // B + s_load_b64 s[6:7], s[0:1], 0x10 // X + s_load_b64 s[8:9], s[0:1], 0x18 // Y + s_load_b64 s[10:11], s[0:1], 0x20 // M, K + s_load_b32 s12, s[0:1], 0x28 // N + s_waitcnt lgkmcnt(0) + + // num_wg_m = ceil(M/32) + s_add_u32 s13, s10, TM - 1 + s_lshr_b32 s13, s13, 5 + + // wg_m = wg_id % num_wg_m, wg_n = wg_id / num_wg_m + s_mov_b32 s14, 0 + s_mov_b32 s15, s20 +.Ldiv: + s_cmp_lt_u32 s15, s13 + s_cbranch_scc1 .Ldiv_done + s_sub_u32 s15, s15, s13 + s_add_u32 s14, s14, 1 + s_branch .Ldiv +.Ldiv_done: + // s15 = wg_m, s14 = wg_n + + // thread decomp + v_and_b32 v1, 31, v0 // thread_m = lid & 31 + v_lshrrev_b32 v2, 5, v0 // thread_n = lid >> 5 + + // global output coords + s_lshl_b32 s16, s15, 5 // wg_m * 32 + s_lshl_b32 s17, s14, 3 // wg_n * 8 + v_add_nc_u32 v3, s16, v1 // global_m + v_add_nc_u32 v4, s17, v2 // global_n + + // load bias into accumulator + v_lshlrev_b32 v5, 2, v3 // global_m * 4 + v_mov_b32 v6, 0 + v_cmp_lt_u32 vcc_lo, v3, s10 + s_and_saveexec_b32 s18, vcc_lo + global_load_b32 v6, v5, s[4:5] + s_mov_b32 exec_lo, s18 + s_waitcnt vmcnt(0) + + // ======== Precompute cooperative load offsets ======== + + // W coop: tile_row = lid/8, tile_col = (lid%8)*4 + v_lshrrev_b32 v7, 3, v0 // tile_row = lid >> 3 + v_and_b32 v8, 7, v0 // lid & 7 + v_lshlrev_b32 v8, 2, v8 // tile_col = (lid & 7) * 4 + + // W global byte offset for k_tile=0: + // ((wg_m*32 + tile_row) * K + tile_col) * 4 + v_add_nc_u32 v9, s16, v7 // wg_m*32 + tile_row + v_mul_lo_u32 v9, v9, s11 // * K + v_add_nc_u32 v9, v9, v8 // + tile_col + v_lshlrev_b32 v9, 2, v9 // * 4 bytes + // v9 = W global load voffset (running, += TK*4 each iter) + + // W LDS store offset = lid * 16 (4 floats * 4 bytes) + v_lshlrev_b32 v10, 4, v0 + + // X coop: x_row = lid/32 = v2, x_col = lid%32 = v1 + // X global byte offset for k_tile=0: + // ((wg_n*8 + x_row) * K + x_col) * 4 + v_add_nc_u32 v11, s17, v2 // wg_n*8 + lid/32 + v_mul_lo_u32 v11, v11, s11 // * K + v_add_nc_u32 v11, v11, v1 // + lid%32 + v_lshlrev_b32 v11, 2, v11 // * 4 bytes + // v11 = X global load voffset (running, += TK*4 each iter) + + // X LDS store offset = LDS_X + lid * 4 + v_lshlrev_b32 v12, 2, v0 + v_add_nc_u32 v12, LDS_X, v12 + + // Compute-phase LDS read bases + v_lshlrev_b32 v13, 7, v1 // W: thread_m * 128 + v_lshlrev_b32 v14, 7, v2 + v_add_nc_u32 v14, LDS_X, v14 // X: LDS_X + thread_n * 128 + + // ======== Tile loop over K ======== + s_mov_b32 s18, 0 // k_tile = 0 + +.Ltile_loop: + s_cmp_ge_u32 s18, s11 // k_tile >= K? + s_cbranch_scc1 .Ltile_done + + // Phase 1: cooperative global → LDS + global_load_b128 v[15:18], v9, s[2:3] // W: 4 consecutive floats + global_load_b32 v19, v11, s[6:7] // X: 1 float + s_waitcnt vmcnt(0) + + ds_store_b128 v10, v[15:18] // W → LDS + ds_store_b32 v12, v19 // X → LDS + s_waitcnt lgkmcnt(0) + s_barrier + + // Phase 2: compute 32 FMAs from LDS, unrolled 4x per block (8 blocks) + + // block 0: tk=0..3 + ds_load_b128 v[15:18], v13 offset:0 + ds_load_b128 v[20:23], v14 offset:0 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 1: tk=4..7 + ds_load_b128 v[15:18], v13 offset:16 + ds_load_b128 v[20:23], v14 offset:16 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 2: tk=8..11 + ds_load_b128 v[15:18], v13 offset:32 + ds_load_b128 v[20:23], v14 offset:32 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 3: tk=12..15 + ds_load_b128 v[15:18], v13 offset:48 + ds_load_b128 v[20:23], v14 offset:48 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 4: tk=16..19 + ds_load_b128 v[15:18], v13 offset:64 + ds_load_b128 v[20:23], v14 offset:64 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 5: tk=20..23 + ds_load_b128 v[15:18], v13 offset:80 + ds_load_b128 v[20:23], v14 offset:80 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 6: tk=24..27 + ds_load_b128 v[15:18], v13 offset:96 + ds_load_b128 v[20:23], v14 offset:96 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 7: tk=28..31 + ds_load_b128 v[15:18], v13 offset:112 + ds_load_b128 v[20:23], v14 offset:112 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + s_barrier + + // advance offsets to next K tile + v_add_nc_u32 v9, v9, TK * 4 // W += 128 bytes + v_add_nc_u32 v11, v11, TK * 4 // X += 128 bytes + s_add_u32 s18, s18, TK + s_branch .Ltile_loop + +.Ltile_done: + // store Y[global_n][global_m] with bounds check + v_cmp_lt_u32 vcc_lo, v3, s10 // global_m < M + v_cmp_lt_u32 s19, v4, s12 // global_n < N + s_and_b32 s19, vcc_lo, s19 + s_and_saveexec_b32 s20, s19 + s_cbranch_execz .Ldone + + v_mul_lo_u32 v15, v4, s10 // global_n * M + v_add_nc_u32 v15, v15, v3 // + global_m + v_lshlrev_b32 v15, 2, v15 // * 4 bytes + global_store_b32 v15, v6, s[8:9] + s_waitcnt vmcnt(0) + +.Ldone: + s_endpgm + +// ======== Kernel descriptor ======== +.rodata +.p2align 6 +.amdhsa_kernel matmul + .amdhsa_group_segment_fixed_size LDS_SZ + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 48 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 24 + .amdhsa_next_free_sgpr 21 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: matmul + .symbol: matmul.kd + .kernarg_segment_size: 48 + .group_segment_fixed_size: 5120 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 21 + .vgpr_count: 24 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 16, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 24, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 32, .value_kind: by_value } + - { .size: 4, .offset: 36, .value_kind: by_value } + - { .size: 4, .offset: 40, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/matmul_blocked.co b/src/kernels/matmul_blocked.co new file mode 100755 index 0000000..f69f6a7 Binary files /dev/null and b/src/kernels/matmul_blocked.co differ diff --git a/src/kernels/matmul_blocked.s b/src/kernels/matmul_blocked.s new file mode 100644 index 0000000..00e0a63 --- /dev/null +++ b/src/kernels/matmul_blocked.s @@ -0,0 +1,338 @@ +// rdna3 register-blocked LDS matmul: Y = X * W^T + B +// +// 4x4 output tile per thread: each thread computes 16 elements. +// TM=128, TN=32, TK=8, 256 threads/WG. +// threads_m = 128/4 = 32, threads_n = 32/4 = 8 +// thread_m = lid % 32, thread_n = lid / 32 +// +// LDS layout (TRANSPOSED for vectorized reads): +// W: W[TK][TM] at offset 0 — 8*128*4 = 4096 bytes +// X: X[TK][TN] at offset 4096 — 8*32*4 = 1024 bytes +// Total: 5120 bytes +// +// Cooperative load (per thread, 4 W elements + 1 X element): +// W: 128*8 = 1024 elts / 256 threads = 4 per thread +// tile_row = lid/2, tile_col = (lid%2)*4 +// global: W[wg_m*128 + tile_row][k_tile + tile_col .. +3] +// LDS (transposed): W[tile_col+i][tile_row] for i=0..3 +// = 4 scattered ds_store_b32 (stride = TM*4 = 512 bytes) +// X: 32*8 = 256 elts / 256 threads = 1 per thread +// x_row = lid/8, x_col = lid%8 +// global: X[wg_n*32 + x_row][k_tile + x_col] +// LDS (transposed): X[x_col][x_row] at LDS_X + x_col*TN*4 + x_row*4 +// = 1 ds_store_b32 +// +// Compute phase (per thread, per tk step): +// W: ds_load_b128 reads W[tk][thread_m*4 .. thread_m*4+3] (contiguous after transpose) +// X: ds_load_b128 reads X[tk][thread_n*4 .. thread_n*4+3] (contiguous after transpose) +// 16 FMAs: outer product of 4 W values * 4 X values +// 2 LDS reads per 16 FMAs = 8x better than non-blocked kernel +// +// dispatch: grid.x = ceil(M/128)*ceil(N/32), block.x = 256 + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.set TM, 128 +.set TN, 32 +.set TK, 8 +.set BM, 4 // block size per thread in M +.set BN, 4 // block size per thread in N +.set LDS_X, 4096 // W: TK*TM*4 = 8*128*4 = 4096 +.set LDS_SZ, 5120 // + X: TK*TN*4 = 8*32*4 = 1024 + +.text +.globl matmul_blocked +.p2align 8 +.type matmul_blocked, @function +matmul_blocked: + s_mov_b32 s20, s2 // save wg_id + + s_load_b64 s[2:3], s[0:1], 0x00 // W + s_load_b64 s[4:5], s[0:1], 0x08 // B + s_load_b64 s[6:7], s[0:1], 0x10 // X + s_load_b64 s[8:9], s[0:1], 0x18 // Y + s_load_b64 s[10:11], s[0:1], 0x20 // M, K + s_load_b32 s12, s[0:1], 0x28 // N + s_waitcnt lgkmcnt(0) + + // num_wg_m = ceil(M/128) + s_add_u32 s13, s10, TM - 1 + s_lshr_b32 s13, s13, 7 // /128 + + // wg_m = wg_id % num_wg_m, wg_n = wg_id / num_wg_m + s_mov_b32 s14, 0 + s_mov_b32 s15, s20 +.Ldiv: + s_cmp_lt_u32 s15, s13 + s_cbranch_scc1 .Ldiv_done + s_sub_u32 s15, s15, s13 + s_add_u32 s14, s14, 1 + s_branch .Ldiv +.Ldiv_done: + // s15 = wg_m, s14 = wg_n + + // thread decomp: 32 threads in M, 8 in N + v_and_b32 v1, 31, v0 // thread_m = lid & 31 + v_lshrrev_b32 v2, 5, v0 // thread_n = lid >> 5 + + // global output coords (base of 4x4 block) + s_lshl_b32 s16, s15, 7 // wg_m * 128 + s_lshl_b32 s17, s14, 5 // wg_n * 32 + v_lshlrev_b32 v3, 2, v1 // thread_m * 4 = base_m offset + v_add_nc_u32 v3, s16, v3 // global_m_base = wg_m*128 + thread_m*4 + v_lshlrev_b32 v4, 2, v2 // thread_n * 4 = base_n offset + v_add_nc_u32 v4, s17, v4 // global_n_base = wg_n*32 + thread_n*4 + + // ======== Initialize 16 accumulators with bias ======== + // acc[i][j] for i=0..3, j=0..3 in v16..v31 + // acc[i][j] = B[global_m_base + i] + // Load 4 bias values + v_mov_b32 v16, 0 + v_mov_b32 v17, 0 + v_mov_b32 v18, 0 + v_mov_b32 v19, 0 + v_mov_b32 v20, 0 + v_mov_b32 v21, 0 + v_mov_b32 v22, 0 + v_mov_b32 v23, 0 + v_mov_b32 v24, 0 + v_mov_b32 v25, 0 + v_mov_b32 v26, 0 + v_mov_b32 v27, 0 + v_mov_b32 v28, 0 + v_mov_b32 v29, 0 + v_mov_b32 v30, 0 + v_mov_b32 v31, 0 + + // load B[global_m_base+0..3] + v_lshlrev_b32 v5, 2, v3 // global_m_base * 4 + v_cmp_lt_u32 vcc_lo, v3, s10 // bounds check + s_and_saveexec_b32 s18, vcc_lo + global_load_b128 v[16:19], v5, s[4:5] // B[m+0..3] → acc[0..3][0] + s_mov_b32 exec_lo, s18 + s_waitcnt vmcnt(0) + // Copy bias to all 4 N columns: acc[i][j] = B[m+i] for j=0..3 + v_mov_b32 v20, v16 // acc[0][1] = B[m+0] + v_mov_b32 v24, v16 // acc[0][2] + v_mov_b32 v28, v16 // acc[0][3] + v_mov_b32 v21, v17 // acc[1][1] = B[m+1] + v_mov_b32 v25, v17 // acc[1][2] + v_mov_b32 v29, v17 // acc[1][3] + v_mov_b32 v22, v18 // acc[2][1] + v_mov_b32 v26, v18 // acc[2][2] + v_mov_b32 v30, v18 // acc[2][3] + v_mov_b32 v23, v19 // acc[3][1] + v_mov_b32 v27, v19 // acc[3][2] + v_mov_b32 v31, v19 // acc[3][3] + + // ======== Precompute cooperative load offsets ======== + + // W coop: tile_row = lid/2 (0..127), tile_col = (lid%2)*4 (0 or 4) + v_lshrrev_b32 v5, 1, v0 // tile_row = lid >> 1 + v_and_b32 v6, 1, v0 // lid & 1 + v_lshlrev_b32 v6, 2, v6 // tile_col = (lid&1)*4 + + // W global byte offset for k_tile=0: + // ((wg_m*128 + tile_row) * K + tile_col) * 4 + v_add_nc_u32 v7, s16, v5 // wg_m*128 + tile_row + v_mul_lo_u32 v7, v7, s11 // * K + v_add_nc_u32 v7, v7, v6 // + tile_col + v_lshlrev_b32 v7, 2, v7 // * 4 bytes + // v7 = W global load voffset (running, += TK*4 each iter) + + // W LDS store offsets (transposed): W[tile_col+i][tile_row] + // base = tile_col * TM * 4 + tile_row * 4 + v_mul_lo_u32 v8, v6, TM // tile_col * 128 + v_lshlrev_b32 v8, 2, v8 // * 4 bytes + v_lshlrev_b32 v9, 2, v5 // tile_row * 4 + v_add_nc_u32 v8, v8, v9 // base LDS offset for W store + // stride between consecutive tile_col values = TM*4 = 512 + + // X coop: x_row = lid/8 (0..31), x_col = lid%8 (0..7) + v_lshrrev_b32 v9, 3, v0 // x_row = lid >> 3 + v_and_b32 v10, 7, v0 // x_col = lid & 7 + + // X global byte offset for k_tile=0: + // ((wg_n*32 + x_row) * K + x_col) * 4 + v_add_nc_u32 v11, s17, v9 // wg_n*32 + x_row + v_mul_lo_u32 v11, v11, s11 // * K + v_add_nc_u32 v11, v11, v10 // + x_col + v_lshlrev_b32 v11, 2, v11 // * 4 bytes + // v11 = X global load voffset (running, += TK*4 each iter) + + // X LDS store offset (transposed): X[x_col][x_row] + // = LDS_X + x_col * TN * 4 + x_row * 4 + v_mul_lo_u32 v12, v10, TN // x_col * 32 + v_lshlrev_b32 v12, 2, v12 // * 4 + v_lshlrev_b32 v13, 2, v9 // x_row * 4 + v_add_nc_u32 v12, v12, v13 + v_add_nc_u32 v12, LDS_X, v12 // + LDS_X + + // Compute-phase LDS read bases (transposed layout) + // W[tk][thread_m*4+0..3]: base = tk * TM * 4 + thread_m * 4 * 4 + // = tk * 512 + thread_m * 16 + // We use offset for tk, so base = thread_m * 16 + v_lshlrev_b32 v13, 4, v1 // thread_m * 16 + + // X[tk][thread_n*4+0..3]: base = LDS_X + tk * TN * 4 + thread_n * 4 * 4 + // = LDS_X + tk * 128 + thread_n * 16 + v_lshlrev_b32 v14, 4, v2 // thread_n * 16 + v_add_nc_u32 v14, LDS_X, v14 // + LDS_X + + // ======== Tile loop over K (with prefetch) ======== + // Prefetch hides global memory latency by issuing the next tile's + // load during the current tile's compute phase. + // v[40:43] = prefetch W, v44 = prefetch X + // v[32:35] = LDS W read, v[36:39] = LDS X read + s_mov_b32 s18, 0 // k_tile = 0 + + // Prologue: load first tile into prefetch regs + global_load_b128 v[40:43], v7, s[2:3] // W tile 0 + global_load_b32 v44, v11, s[6:7] // X tile 0 + +.Ltile_loop: + // Wait for current tile's global data (first iter: prologue, then: prefetch) + s_waitcnt vmcnt(0) + + // Store prefetched data to LDS (transposed) + ds_store_b32 v8, v40 // W[tile_col+0][tile_row] + ds_store_b32 v8, v41 offset:512 // W[tile_col+1][tile_row] + ds_store_b32 v8, v42 offset:1024 // W[tile_col+2][tile_row] + ds_store_b32 v8, v43 offset:1536 // W[tile_col+3][tile_row] + ds_store_b32 v12, v44 // X[x_col][x_row] + s_waitcnt lgkmcnt(0) + s_barrier + + // Compute from LDS — 8 k-steps × 16 FMAs = 128 FMAs + // Prefetch issued AFTER first LDS reads to not stall the barrier→compute path + + // tk=0 + ds_load_b128 v[32:35], v13 offset:0 + ds_load_b128 v[36:39], v14 offset:0 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v16, v32, v36 + v_fmac_f32 v17, v33, v36 + v_fmac_f32 v18, v34, v36 + v_fmac_f32 v19, v35, v36 + v_fmac_f32 v20, v32, v37 + v_fmac_f32 v21, v33, v37 + v_fmac_f32 v22, v34, v37 + v_fmac_f32 v23, v35, v37 + v_fmac_f32 v24, v32, v38 + v_fmac_f32 v25, v33, v38 + v_fmac_f32 v26, v34, v38 + v_fmac_f32 v27, v35, v38 + v_fmac_f32 v28, v32, v39 + v_fmac_f32 v29, v33, v39 + v_fmac_f32 v30, v34, v39 + v_fmac_f32 v31, v35, v39 + + // Issue prefetch after first tk step — overlaps with tk=1..7 compute + v_add_nc_u32 v7, v7, TK * 4 // W global += 32 bytes + v_add_nc_u32 v11, v11, TK * 4 // X global += 32 bytes + s_add_u32 s18, s18, TK + global_load_b128 v[40:43], v7, s[2:3] // prefetch next W + global_load_b32 v44, v11, s[6:7] // prefetch next X + + // tk=1..7 + .irp TK_OFF, 512, 1024, 1536, 2048, 2560, 3072, 3584 + ds_load_b128 v[32:35], v13 offset:\TK_OFF + ds_load_b128 v[36:39], v14 offset:(\TK_OFF / 4) + s_waitcnt lgkmcnt(0) + v_fmac_f32 v16, v32, v36 + v_fmac_f32 v17, v33, v36 + v_fmac_f32 v18, v34, v36 + v_fmac_f32 v19, v35, v36 + v_fmac_f32 v20, v32, v37 + v_fmac_f32 v21, v33, v37 + v_fmac_f32 v22, v34, v37 + v_fmac_f32 v23, v35, v37 + v_fmac_f32 v24, v32, v38 + v_fmac_f32 v25, v33, v38 + v_fmac_f32 v26, v34, v38 + v_fmac_f32 v27, v35, v38 + v_fmac_f32 v28, v32, v39 + v_fmac_f32 v29, v33, v39 + v_fmac_f32 v30, v34, v39 + v_fmac_f32 v31, v35, v39 + .endr + + s_barrier + + // Loop if prefetched tile is valid + s_cmp_lt_u32 s18, s11 // s18 < K? + s_cbranch_scc1 .Ltile_loop + +.Ltile_done: + // ======== Store 16 output elements ======== + // Y[global_n_base+j][global_m_base+i] for i=0..3, j=0..3 + // Y layout: row-major, Y[n][m], stride = M + // offset = (global_n_base + j) * M + (global_m_base + i) + + // Compute base offset: global_n_base * M + global_m_base + v_mul_lo_u32 v5, v4, s10 // global_n_base * M + v_add_nc_u32 v5, v5, v3 // + global_m_base + v_lshlrev_b32 v5, 2, v5 // * 4 bytes + + // Store row j=0: acc[0..3][0] = v16..v19 + global_store_b128 v5, v[16:19], s[8:9] + + // Row j=1: offset += M*4 + s_lshl_b32 s19, s10, 2 // M * 4 + v_add_nc_u32 v5, v5, s19 + global_store_b128 v5, v[20:23], s[8:9] + + // Row j=2 + v_add_nc_u32 v5, v5, s19 + global_store_b128 v5, v[24:27], s[8:9] + + // Row j=3 + v_add_nc_u32 v5, v5, s19 + global_store_b128 v5, v[28:31], s[8:9] + + s_waitcnt vmcnt(0) + s_endpgm + +// ======== Kernel descriptor ======== +.rodata +.p2align 6 +.amdhsa_kernel matmul_blocked + .amdhsa_group_segment_fixed_size LDS_SZ + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 48 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 48 + .amdhsa_next_free_sgpr 21 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: matmul_blocked + .symbol: matmul_blocked.kd + .kernarg_segment_size: 48 + .group_segment_fixed_size: 5120 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 21 + .vgpr_count: 48 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 16, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 24, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 32, .value_kind: by_value } + - { .size: 4, .offset: 36, .value_kind: by_value } + - { .size: 4, .offset: 40, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/matmul_dbg.co b/src/kernels/matmul_dbg.co new file mode 100755 index 0000000..c41313c Binary files /dev/null and b/src/kernels/matmul_dbg.co differ diff --git a/src/kernels/matmul_dbg.s b/src/kernels/matmul_dbg.s new file mode 100644 index 0000000..ae0173f --- /dev/null +++ b/src/kernels/matmul_dbg.s @@ -0,0 +1,282 @@ +// rdna3 LDS-tiled matmul: Y = X * W^T + B +// +// Each workgroup computes a 32x8 tile of Y. +// 256 threads/WG: thread_m = lid%32, thread_n = lid/32 +// Tiles K in chunks of TK=32. Each tile iteration: +// 1. Cooperative load W[32][32] and X[8][32] to LDS +// 2. barrier +// 3. Each thread reads its W row and X row from LDS, does 32 FMAs +// 4. barrier +// +// LDS layout: W at 0 (4096B), X at 4096 (1024B), total 5120B +// +// Cooperative load assignment (W: 1024 elts / 256 threads = 4 each): +// tile_row = lid/8, tile_col = (lid%8)*4 +// global_load_b128 loads W[tile_row][tile_col..+3] +// ds_store_b128 at LDS offset lid*16 +// +// Cooperative load assignment (X: 256 elts / 256 threads = 1 each): +// x_row = lid/32, x_col = lid%32 +// global_load_b32 loads X[x_row][x_col] +// ds_store_b32 at LDS offset 4096 + lid*4 +// +// SGPR: s[0:1]=kernarg, s2=TGID_X +// kernargs (48B): W(u64) B(u64) X(u64) Y(u64) M(u32) K(u32) N(u32) +// dispatch: grid.x = ceil(M/32)*ceil(N/8)*256, block.x = 256 + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.set TM, 32 +.set TN, 8 +.set TK, 32 +.set LDS_X, 4096 // W uses 0..4095, X uses 4096..5119 +.set LDS_SZ, 5120 + +.text +.globl matmul_dbg +.p2align 8 +.type matmul, @function +matmul_dbg: + s_mov_b32 s20, s2 // save wg_id + + s_load_b64 s[2:3], s[0:1], 0x00 // W + s_load_b64 s[4:5], s[0:1], 0x08 // B + s_load_b64 s[6:7], s[0:1], 0x10 // X + s_load_b64 s[8:9], s[0:1], 0x18 // Y + s_load_b64 s[10:11], s[0:1], 0x20 // M, K + s_load_b32 s12, s[0:1], 0x28 // N + s_waitcnt lgkmcnt(0) + + // num_wg_m = ceil(M/32) + s_add_u32 s13, s10, TM - 1 + s_lshr_b32 s13, s13, 5 + + // wg_m = wg_id % num_wg_m, wg_n = wg_id / num_wg_m + s_mov_b32 s14, 0 + s_mov_b32 s15, s20 +.Ldiv: + s_cmp_lt_u32 s15, s13 + s_cbranch_scc1 .Ldiv_done + s_sub_u32 s15, s15, s13 + s_add_u32 s14, s14, 1 + s_branch .Ldiv +.Ldiv_done: + // s15 = wg_m, s14 = wg_n + + // thread decomp + v_and_b32 v1, 31, v0 // thread_m = lid & 31 + v_lshrrev_b32 v2, 5, v0 // thread_n = lid >> 5 + + // global output coords + s_lshl_b32 s16, s15, 5 // wg_m * 32 + s_lshl_b32 s17, s14, 3 // wg_n * 8 + v_add_nc_u32 v3, s16, v1 // global_m + v_add_nc_u32 v4, s17, v2 // global_n + + // load bias into accumulator + v_lshlrev_b32 v5, 2, v3 // global_m * 4 + v_mov_b32 v6, 0 + v_cmp_lt_u32 vcc_lo, v3, s10 + s_and_saveexec_b32 s18, vcc_lo + global_load_b32 v6, v5, s[4:5] + s_mov_b32 exec_lo, s18 + // s_waitcnt vmcnt(0) -- no global loads + + // ======== Precompute cooperative load offsets ======== + + // W coop: tile_row = lid/8, tile_col = (lid%8)*4 + v_lshrrev_b32 v7, 3, v0 // tile_row = lid >> 3 + v_and_b32 v8, 7, v0 // lid & 7 + v_lshlrev_b32 v8, 2, v8 // tile_col = (lid & 7) * 4 + + // W global byte offset for k_tile=0: + // ((wg_m*32 + tile_row) * K + tile_col) * 4 + v_add_nc_u32 v9, s16, v7 // wg_m*32 + tile_row + v_mul_lo_u32 v9, v9, s11 // * K + v_add_nc_u32 v9, v9, v8 // + tile_col + v_lshlrev_b32 v9, 2, v9 // * 4 bytes + // v9 = W global load voffset (running, += TK*4 each iter) + + // W LDS store offset = lid * 16 (4 floats * 4 bytes) + v_lshlrev_b32 v10, 4, v0 + + // X coop: x_row = lid/32 = v2, x_col = lid%32 = v1 + // X global byte offset for k_tile=0: + // ((wg_n*8 + x_row) * K + x_col) * 4 + v_add_nc_u32 v11, s17, v2 // wg_n*8 + lid/32 + v_mul_lo_u32 v11, v11, s11 // * K + v_add_nc_u32 v11, v11, v1 // + lid%32 + v_lshlrev_b32 v11, 2, v11 // * 4 bytes + // v11 = X global load voffset (running, += TK*4 each iter) + + // X LDS store offset = LDS_X + lid * 4 + v_lshlrev_b32 v12, 2, v0 + v_add_nc_u32 v12, LDS_X, v12 + + // Compute-phase LDS read bases + v_lshlrev_b32 v13, 7, v1 // W: thread_m * 128 + v_lshlrev_b32 v14, 7, v2 + v_add_nc_u32 v14, LDS_X, v14 // X: LDS_X + thread_n * 128 + + // ======== Tile loop over K ======== + s_mov_b32 s18, 0 // k_tile = 0 + +.Ltile_loop: + s_cmp_ge_u32 s18, s11 // k_tile >= K? + s_cbranch_scc1 .Ltile_done + + // Phase 1: cooperative global → LDS + v_mov_b32 v15, 1.0 + v_mov_b32 v16, 1.0 + v_mov_b32 v17, 1.0 + v_mov_b32 v18, 1.0 + v_mov_b32 v19, 1.0 + // s_waitcnt vmcnt(0) -- no global loads + + ds_store_b128 v10, v[15:18] // W → LDS + ds_store_b32 v12, v19 // X → LDS + s_waitcnt lgkmcnt(0) + s_barrier + + // Phase 2: compute 32 FMAs from LDS, unrolled 4x per block (8 blocks) + + // block 0: tk=0..3 + ds_load_b128 v[15:18], v13 offset:0 + ds_load_b128 v[20:23], v14 offset:0 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 1: tk=4..7 + ds_load_b128 v[15:18], v13 offset:16 + ds_load_b128 v[20:23], v14 offset:16 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 2: tk=8..11 + ds_load_b128 v[15:18], v13 offset:32 + ds_load_b128 v[20:23], v14 offset:32 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 3: tk=12..15 + ds_load_b128 v[15:18], v13 offset:48 + ds_load_b128 v[20:23], v14 offset:48 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 4: tk=16..19 + ds_load_b128 v[15:18], v13 offset:64 + ds_load_b128 v[20:23], v14 offset:64 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 5: tk=20..23 + ds_load_b128 v[15:18], v13 offset:80 + ds_load_b128 v[20:23], v14 offset:80 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 6: tk=24..27 + ds_load_b128 v[15:18], v13 offset:96 + ds_load_b128 v[20:23], v14 offset:96 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + // block 7: tk=28..31 + ds_load_b128 v[15:18], v13 offset:112 + ds_load_b128 v[20:23], v14 offset:112 + s_waitcnt lgkmcnt(0) + v_fmac_f32 v6, v15, v20 + v_fmac_f32 v6, v16, v21 + v_fmac_f32 v6, v17, v22 + v_fmac_f32 v6, v18, v23 + + s_barrier + + // advance offsets to next K tile + v_add_nc_u32 v9, v9, TK * 4 // W += 128 bytes + v_add_nc_u32 v11, v11, TK * 4 // X += 128 bytes + s_add_u32 s18, s18, TK + s_branch .Ltile_loop + +.Ltile_done: + // store Y[global_n][global_m] with bounds check + v_cmp_lt_u32 vcc_lo, v3, s10 // global_m < M + v_cmp_lt_u32 s19, v4, s12 // global_n < N + s_and_b32 s19, vcc_lo, s19 + s_and_saveexec_b32 s20, s19 + s_cbranch_execz .Ldone + + v_mul_lo_u32 v15, v4, s10 // global_n * M + v_add_nc_u32 v15, v15, v3 // + global_m + v_lshlrev_b32 v15, 2, v15 // * 4 bytes + global_store_b32 v15, v6, s[8:9] + // s_waitcnt vmcnt(0) -- no global loads + +.Ldone: + s_endpgm + +// ======== Kernel descriptor ======== +.rodata +.p2align 6 +.amdhsa_kernel matmul_dbg + .amdhsa_group_segment_fixed_size LDS_SZ + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 48 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 24 + .amdhsa_next_free_sgpr 21 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: matmul_dbg + .symbol: matmul_dbg.kd + .kernarg_segment_size: 48 + .group_segment_fixed_size: 5120 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 21 + .vgpr_count: 24 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 16, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 24, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 32, .value_kind: by_value } + - { .size: 4, .offset: 36, .value_kind: by_value } + - { .size: 4, .offset: 40, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/matmul_small.co b/src/kernels/matmul_small.co new file mode 100755 index 0000000..67c4736 Binary files /dev/null and b/src/kernels/matmul_small.co differ diff --git a/src/kernels/matmul_small.s b/src/kernels/matmul_small.s new file mode 100644 index 0000000..a9bb720 --- /dev/null +++ b/src/kernels/matmul_small.s @@ -0,0 +1,310 @@ +// TM=32 TT2x2 TK=8 wave32, PLR, single-buffer LDS +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 +.set TM, 32 +.set TN, 32 +.set TK, 8 +.set LDS_X, 1024 +.set LDS_SZ, 2048 +.text +.globl matmul_small +.p2align 8 +.type matmul_small, @function +matmul_small: + s_mov_b32 s20, s2 + s_load_b64 s[2:3], s[0:1], 0x00 + s_load_b64 s[4:5], s[0:1], 0x08 + s_load_b64 s[6:7], s[0:1], 0x10 + s_load_b64 s[8:9], s[0:1], 0x18 + s_load_b64 s[10:11], s[0:1], 0x20 + s_load_b32 s12, s[0:1], 0x28 + s_waitcnt lgkmcnt(0) + s_add_u32 s13, s10, TM - 1 + s_lshr_b32 s13, s13, 5 + s_mov_b32 s14, 0 + s_mov_b32 s15, s20 +.Ldiv: + s_cmp_lt_u32 s15, s13 + s_cbranch_scc1 .Ldiv_done + s_sub_u32 s15, s15, s13 + s_add_u32 s14, s14, 1 + s_branch .Ldiv +.Ldiv_done: + v_and_b32 v1, 15, v0 + v_lshrrev_b32 v2, 4, v0 + s_lshl_b32 s16, s15, 5 + s_lshl_b32 s17, s14, 5 + v_lshlrev_b32 v3, 1, v1 + v_add_nc_u32 v3, s16, v3 + v_lshlrev_b32 v4, 1, v2 + v_add_nc_u32 v4, s17, v4 + v_mov_b32 v16, 0 + v_mov_b32 v17, 0 + v_mov_b32 v18, 0 + v_mov_b32 v19, 0 + v_lshlrev_b32 v5, 2, v3 + v_cmp_lt_u32 vcc_lo, v3, s10 + s_and_saveexec_b32 s18, vcc_lo + global_load_b64 v[16:17], v5, s[4:5] + s_mov_b32 exec_lo, s18 + s_waitcnt vmcnt(0) + v_mov_b32 v18, v16 + v_mov_b32 v19, v17 + // W coop (col-major) + v_lshrrev_b32 v5, 5, v0 + v_and_b32 v6, 31, v0 + v_mul_lo_u32 v7, v5, s10 + v_add_nc_u32 v7, v7, s16 + v_add_nc_u32 v7, v7, v6 + v_lshlrev_b32 v7, 2, v7 + v_mul_lo_u32 v8, v5, TM + v_add_nc_u32 v8, v8, v6 + v_lshlrev_b32 v8, 2, v8 + s_lshl_b32 s22, s10, 5 + // X coop (coalesced) + v_lshrrev_b32 v5, 3, v0 + v_and_b32 v6, 7, v0 + v_add_nc_u32 v9, s17, v5 + v_mul_lo_u32 v9, v9, s11 + v_add_nc_u32 v9, v9, v6 + v_lshlrev_b32 v9, 2, v9 + v_mul_lo_u32 v10, v6, TN + v_add_nc_u32 v10, v10, v5 + v_lshlrev_b32 v10, 2, v10 + v_add_nc_u32 v10, LDS_X, v10 + // LDS read bases + v_lshlrev_b32 v11, 3, v1 + v_lshlrev_b32 v12, 3, v2 + v_add_nc_u32 v12, LDS_X, v12 + // Prologue + s_mov_b32 s18, 0 + global_load_b32 v20, v7, s[2:3] + global_load_b32 v21, v9, s[6:7] + s_waitcnt vmcnt(0) + ds_store_b32 v8, v20 + ds_store_b32 v10, v21 + s_waitcnt lgkmcnt(0) + s_barrier + v_add_nc_u32 v7, v7, s22 + v_add_nc_u32 v9, v9, TK * 4 + s_add_u32 s18, s18, TK + s_cmp_ge_u32 s18, s11 + s_cbranch_scc1 .Llast_tile +.Ltile_loop: + // Prefetch tk=0 + ds_load_2addr_b32 v[22:23], v11 offset0:0 offset1:1 + ds_load_2addr_b32 v[24:25], v12 offset0:0 offset1:1 + global_load_b32 v20, v7, s[2:3] + global_load_b32 v21, v9, s[6:7] + s_waitcnt lgkmcnt(0) + // tk=0 + ds_load_2addr_b32 v[26:27], v11 offset0:(1*32) offset1:(1*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(1*32) offset1:(1*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + // tk=1 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[22:23], v11 offset0:(2*32) offset1:(2*32+1) + ds_load_2addr_b32 v[24:25], v12 offset0:(2*32) offset1:(2*32+1) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + // tk=2 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[26:27], v11 offset0:(3*32) offset1:(3*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(3*32) offset1:(3*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + // tk=3 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[22:23], v11 offset0:(4*32) offset1:(4*32+1) + ds_load_2addr_b32 v[24:25], v12 offset0:(4*32) offset1:(4*32+1) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + // tk=4 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[26:27], v11 offset0:(5*32) offset1:(5*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(5*32) offset1:(5*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + // tk=5 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[22:23], v11 offset0:(6*32) offset1:(6*32+1) + ds_load_2addr_b32 v[24:25], v12 offset0:(6*32) offset1:(6*32+1) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + // tk=6 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[26:27], v11 offset0:(7*32) offset1:(7*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(7*32) offset1:(7*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + // tk=7 + s_waitcnt lgkmcnt(0) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + // Store next tile + s_waitcnt vmcnt(0) + ds_store_b32 v8, v20 + ds_store_b32 v10, v21 + s_waitcnt lgkmcnt(0) + s_barrier + v_add_nc_u32 v7, v7, s22 + v_add_nc_u32 v9, v9, TK * 4 + s_add_u32 s18, s18, TK + s_cmp_lt_u32 s18, s11 + s_cbranch_scc1 .Ltile_loop +.Llast_tile: + ds_load_2addr_b32 v[22:23], v11 offset0:0 offset1:1 + ds_load_2addr_b32 v[24:25], v12 offset0:0 offset1:1 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[26:27], v11 offset0:(1*32) offset1:(1*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(1*32) offset1:(1*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[22:23], v11 offset0:(2*32) offset1:(2*32+1) + ds_load_2addr_b32 v[24:25], v12 offset0:(2*32) offset1:(2*32+1) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[26:27], v11 offset0:(3*32) offset1:(3*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(3*32) offset1:(3*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[22:23], v11 offset0:(4*32) offset1:(4*32+1) + ds_load_2addr_b32 v[24:25], v12 offset0:(4*32) offset1:(4*32+1) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[26:27], v11 offset0:(5*32) offset1:(5*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(5*32) offset1:(5*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[22:23], v11 offset0:(6*32) offset1:(6*32+1) + ds_load_2addr_b32 v[24:25], v12 offset0:(6*32) offset1:(6*32+1) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + s_waitcnt lgkmcnt(0) + ds_load_2addr_b32 v[26:27], v11 offset0:(7*32) offset1:(7*32+1) + ds_load_2addr_b32 v[28:29], v12 offset0:(7*32) offset1:(7*32+1) + s_setprio 1 + v_fmac_f32 v16, v22, v24 + v_fmac_f32 v17, v23, v24 + v_fmac_f32 v18, v22, v25 + v_fmac_f32 v19, v23, v25 + s_setprio 0 + s_waitcnt lgkmcnt(0) + s_setprio 1 + v_fmac_f32 v16, v26, v28 + v_fmac_f32 v17, v27, v28 + v_fmac_f32 v18, v26, v29 + v_fmac_f32 v19, v27, v29 + s_setprio 0 + // Store + v_mul_lo_u32 v5, v4, s10 + v_add_nc_u32 v5, v5, v3 + v_lshlrev_b32 v5, 2, v5 + global_store_b64 v5, v[16:17], s[8:9] + s_lshl_b32 s19, s10, 2 + v_add_nc_u32 v5, v5, s19 + global_store_b64 v5, v[18:19], s[8:9] + s_waitcnt vmcnt(0) + s_endpgm +.rodata +.p2align 6 +.amdhsa_kernel matmul_small + .amdhsa_group_segment_fixed_size LDS_SZ + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 48 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 30 + .amdhsa_next_free_sgpr 23 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: matmul_small + .symbol: matmul_small.kd + .kernarg_segment_size: 48 + .group_segment_fixed_size: 2048 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 23 + .vgpr_count: 30 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 16, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 24, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 32, .value_kind: by_value } + - { .size: 4, .offset: 36, .value_kind: by_value } + - { .size: 4, .offset: 40, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/matvec.cl b/src/kernels/matvec.cl new file mode 100644 index 0000000..dd701d9 --- /dev/null +++ b/src/kernels/matvec.cl @@ -0,0 +1,106 @@ +// Matrix-vector multiply: y = W*x + b +// W is [out_dim x in_dim] row-major +// Each workgroup computes one output element +__kernel void matvec( + __global const float* W, + __global const float* b, + __global const float* x, + __global float* y, + uint out_dim, + uint in_dim +) { + uint row = get_global_id(0); + if (row >= out_dim) return; + + float sum = b[row]; + __global const float* w_row = W + row * in_dim; + + // Vectorized inner loop + uint i = 0; + for (; i + 4 <= in_dim; i += 4) { + sum += w_row[i] * x[i]; + sum += w_row[i+1] * x[i+1]; + sum += w_row[i+2] * x[i+2]; + sum += w_row[i+3] * x[i+3]; + } + for (; i < in_dim; i++) { + sum += w_row[i] * x[i]; + } + + y[row] = sum; +} + +// Fused synapse: matvec → GLU → SiLU → layer_norm +// W is [out_dim*2 x in_dim], produces out_dim outputs +// scratch is [out_dim*2] temp space +__kernel void synapse_fused( + __global const float* W, + __global const float* b, + __global const float* x, + __global float* output, + __global float* scratch, + uint out_dim, + uint in_dim +) { + uint row = get_global_id(0); + uint total_rows = out_dim * 2; + if (row >= total_rows) return; + + // Step 1: matvec into scratch + float sum = b[row]; + __global const float* w_row = W + row * in_dim; + for (uint i = 0; i < in_dim; i++) { + sum += w_row[i] * x[i]; + } + scratch[row] = sum; + + barrier(CLK_GLOBAL_MEM_FENCE); + + // Only first out_dim threads continue for GLU + SiLU + if (row >= out_dim) return; + + // Step 2: GLU — output[i] = scratch[i] * sigmoid(scratch[i + out_dim]) + float val = scratch[row]; + float gate = 1.0f / (1.0f + exp(-scratch[row + out_dim])); + float glu_out = val * gate; + + // Step 3: SiLU — x * sigmoid(x) + float silu_out = glu_out / (1.0f + exp(-glu_out)); + + output[row] = silu_out; +} + +// GLU activation +__kernel void glu( + __global const float* input, + __global float* output, + uint half_dim +) { + uint i = get_global_id(0); + if (i >= half_dim) return; + float gate = 1.0f / (1.0f + exp(-input[i + half_dim])); + output[i] = input[i] * gate; +} + +// SiLU (swish) in-place +__kernel void silu( + __global float* x, + uint n +) { + uint i = get_global_id(0); + if (i >= n) return; + float v = x[i]; + x[i] = v / (1.0f + exp(-v)); +} + +// Elementwise add: y[i] += alpha * x[i] +__kernel void axpy( + __global float* y, + __global const float* x, + float alpha, + uint n +) { + uint i = get_global_id(0); + if (i >= n) return; + y[i] += alpha * x[i]; +} diff --git a/src/kernels/matvec.co b/src/kernels/matvec.co new file mode 100755 index 0000000..ec087cf Binary files /dev/null and b/src/kernels/matvec.co differ diff --git a/src/kernels/matvec.s b/src/kernels/matvec.s new file mode 100644 index 0000000..db91cf5 --- /dev/null +++ b/src/kernels/matvec.s @@ -0,0 +1,138 @@ +// rdna3 matvec kernel: y = W*x + b +// kernarg layout (40 bytes, no hidden args): +// +0x00: W pointer (u64) — [out_dim x in_dim] row-major f32 +// +0x08: b pointer (u64) — [out_dim] f32 bias +// +0x10: x pointer (u64) — [in_dim] f32 input +// +0x18: y pointer (u64) — [out_dim] f32 output +// +0x20: out_dim (u32) +// +0x24: in_dim (u32) +// +// each workitem computes one output element (row). +// dispatch: global_size = out_dim, local_size = 1 (or up to 256) +// uses flat_load/flat_store with full 64-bit vgpr addresses (proven pattern) + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.text +.globl matvec +.p2align 8 +.type matvec, @function +matvec: + // s[0:1] = kernarg pointer + // v0 = workitem id + + // load all kernargs + s_load_b64 s[2:3], s[0:1], 0x00 // W ptr + s_load_b64 s[4:5], s[0:1], 0x08 // b ptr + s_load_b64 s[6:7], s[0:1], 0x10 // x ptr + s_load_b64 s[8:9], s[0:1], 0x18 // y ptr + s_load_b64 s[10:11], s[0:1], 0x20 // out_dim(lo) | in_dim(hi) + s_waitcnt lgkmcnt(0) + + // s10 = out_dim, s11 = in_dim (loaded as b64 from offset 0x20) + // bounds check: if v0 >= out_dim, skip + v_cmp_lt_u32 vcc_lo, v0, s10 + s_and_saveexec_b32 s12, vcc_lo + s_cbranch_execz .Ldone + + // row = v0 + // row_byte_off = v0 * in_dim * 4 (byte offset into W for this row) + v_mul_lo_u32 v1, v0, s11 // v1 = row * in_dim (element offset) + v_lshlrev_b32 v1, 2, v1 // v1 = row * in_dim * 4 (byte offset) + + // --- load bias b[row] using flat addressing --- + // addr = b_ptr + row*4 + v_lshlrev_b32 v10, 2, v0 // v10 = row * 4 + v_add_co_u32 v12, vcc_lo, s4, v10 // lo = b_lo + row*4 + v_add_co_ci_u32 v13, vcc_lo, s5, 0, vcc_lo // hi with carry + flat_load_b32 v3, v[12:13] // v3 = b[row] + s_waitcnt vmcnt(0) lgkmcnt(0) + + // v3 = accumulator (initialized to bias) + // loop over in_dim: sum += W[row*in_dim + i] * x[i] + s_mov_b32 s13, 0 // i = 0 + +.Lloop: + s_cmp_ge_u32 s13, s11 // i >= in_dim? + s_cbranch_scc1 .Lloop_done + + // w_byte_off = row_byte_off + i*4 + s_lshl_b32 s14, s13, 2 // s14 = i * 4 + + // --- load W[row][i] via flat --- + // addr = W_ptr + row_byte_off + i*4 + v_add_nc_u32 v4, v1, s14 // v4 = row_byte_off + i*4 + v_add_co_u32 v14, vcc_lo, s2, v4 // lo + v_add_co_ci_u32 v15, vcc_lo, s3, 0, vcc_lo // hi + flat_load_b32 v5, v[14:15] // v5 = W[row][i] + + // --- load x[i] via flat --- + // addr = x_ptr + i*4 + v_mov_b32 v6, s14 // v6 = i*4 + v_add_co_u32 v16, vcc_lo, s6, v6 // lo + v_add_co_ci_u32 v17, vcc_lo, s7, 0, vcc_lo // hi + flat_load_b32 v7, v[16:17] // v7 = x[i] + + s_waitcnt vmcnt(0) lgkmcnt(0) + + // sum += W[row][i] * x[i] + v_fmac_f32 v3, v5, v7 + + // i++ + s_add_u32 s13, s13, 1 + s_branch .Lloop + +.Lloop_done: + // --- store y[row] via flat --- + // addr = y_ptr + row*4 + v_add_co_u32 v18, vcc_lo, s8, v10 // lo = y_lo + row*4 + v_add_co_ci_u32 v19, vcc_lo, s9, 0, vcc_lo // hi + flat_store_b32 v[18:19], v3 + s_waitcnt vmcnt(0) lgkmcnt(0) + +.Ldone: + s_waitcnt vmcnt(0) lgkmcnt(0) + s_endpgm + +// kernel descriptor +.rodata +.p2align 6 +.amdhsa_kernel matvec + .amdhsa_group_segment_fixed_size 0 + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 40 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_next_free_vgpr 20 + .amdhsa_next_free_sgpr 15 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +// AMDGPU metadata for HIP runtime module loading +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: matvec + .symbol: matvec.kd + .kernarg_segment_size: 40 + .group_segment_fixed_size: 0 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 15 + .vgpr_count: 20 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 16, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 24, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 32, .value_kind: by_value } + - { .size: 4, .offset: 36, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/matvec_asm.co b/src/kernels/matvec_asm.co new file mode 100755 index 0000000..f67da03 Binary files /dev/null and b/src/kernels/matvec_asm.co differ diff --git a/src/kernels/superlinear.co b/src/kernels/superlinear.co new file mode 100755 index 0000000..233606f Binary files /dev/null and b/src/kernels/superlinear.co differ diff --git a/src/kernels/superlinear.s b/src/kernels/superlinear.s new file mode 100644 index 0000000..c57625e --- /dev/null +++ b/src/kernels/superlinear.s @@ -0,0 +1,180 @@ +// SuperLinear forward: N independent matvecs with per-neuron weights. +// Y[n*O+o] = B[n*O+o] + sum_k W[n*O*K + o*K + k] * X[n*K + k] +// +// Args: W(ptr), B(ptr), X(ptr), Y(ptr), N(u32), O(u32), K(u32) +// Grid: ceil(N*O / 256) workgroups, 256 threads each. +// Each thread computes one output element. +// K must be multiple of 4 and <= 32. Typical: K=4,8,16. +// +// Memory layout: +// W: [N * O * K] row-major (same as SuperLinear.weights) +// B: [N * O] +// X: [N * K] (trace, flat arena) +// Y: [N * O] (output) + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.text +.globl superlinear_fwd +.p2align 8 +.type superlinear_fwd, @function +superlinear_fwd: + // s[0:1] = kernarg_segment_ptr + // s2 = workgroup_id_x + + // Load kernel args (7 args = 48 bytes) + s_load_b64 s[4:5], s[0:1], 0x00 // W ptr + s_load_b64 s[6:7], s[0:1], 0x08 // B ptr + s_load_b64 s[8:9], s[0:1], 0x10 // X ptr + s_load_b64 s[10:11], s[0:1], 0x18 // Y ptr + s_load_b32 s12, s[0:1], 0x20 // N (n_neurons) + s_load_b32 s13, s[0:1], 0x24 // O (out_per) + s_load_b32 s14, s[0:1], 0x28 // K (in_per) + s_waitcnt lgkmcnt(0) + + // Global ID = workgroup_id * 256 + local_id + s_lshl_b32 s2, s2, 8 // s2 = workgroup_id * 256 + v_add_nc_u32 v1, s2, v0 // v1 = global_id + + // Total outputs = N * O + s_mul_i32 s15, s12, s13 // s15 = N * O + v_cmp_lt_u32 vcc_lo, v1, s15 // bounds check + s_and_saveexec_b32 s16, vcc_lo + s_cbranch_execz .Lexit + + // Compute neuron = global_id / O, out_idx = global_id % O + // Use FP32 reciprocal for approximate division, then correct. + v_cvt_f32_u32 v4, s13 // v4 = float(O) + v_rcp_iflag_f32 v4, v4 // v4 = 1.0/O (approx) + v_cvt_f32_u32 v5, v1 // v5 = float(gid) + v_mul_f32 v5, v5, v4 // v5 = gid / O (float approx) + v_cvt_u32_f32 v2, v5 // v2 = neuron (truncated) + + // Fix overshoot: if neuron * O > gid, decrement + v_mul_lo_u32 v4, v2, s13 + v_cmp_gt_u32 vcc_lo, v4, v1 + v_cndmask_b32 v5, 0, 1, vcc_lo + v_sub_nc_u32 v2, v2, v5 + // Fix undershoot: if (neuron+1)*O <= gid, increment + v_add_nc_u32 v5, v2, 1 + v_mul_lo_u32 v5, v5, s13 + v_cmp_le_u32 vcc_lo, v5, v1 + v_cndmask_b32 v5, 0, 1, vcc_lo + v_add_nc_u32 v2, v2, v5 + v_mul_lo_u32 v4, v2, s13 // v4 = neuron * O + + // out_idx = gid - neuron * O + v_sub_nc_u32 v3, v1, v4 // v3 = out_idx + + // W byte offset = (neuron*O*K + out_idx*K) * 4 + // = ((neuron*O + out_idx) * K) * 4 + v_add_nc_u32 v5, v4, v3 // neuron*O + out_idx + v_mul_lo_u32 v5, v5, s14 // * K + v_lshlrev_b32 v5, 2, v5 // * 4 bytes + + // X byte offset = neuron * K * 4 + v_mul_lo_u32 v6, v2, s14 + v_lshlrev_b32 v6, 2, v6 + + // B byte offset = (neuron*O + out_idx) * 4 + v_add_nc_u32 v7, v4, v3 + v_lshlrev_b32 v7, 2, v7 + + // Load bias + global_load_b32 v20, v7, s[6:7] + + // Dot product: accumulate in v21 + v_mov_b32 v21, 0 // acc = 0.0 + + // Loop over K in steps of 4 (vectorized loads) + s_mov_b32 s17, 0 +.Ldot4_loop: + s_add_u32 s18, s17, 4 + s_cmp_gt_u32 s18, s14 // if counter+4 > K, done with vec loop + s_cbranch_scc1 .Ldot4_done + + // Load 4 floats from W and X + global_load_b128 v[12:15], v5, s[4:5] // W[0..3] + global_load_b128 v[16:19], v6, s[8:9] // X[0..3] + s_waitcnt vmcnt(0) + + v_fmac_f32 v21, v12, v16 + v_fmac_f32 v21, v13, v17 + v_fmac_f32 v21, v14, v18 + v_fmac_f32 v21, v15, v19 + + v_add_nc_u32 v5, v5, 16 // advance W ptr by 4 floats + v_add_nc_u32 v6, v6, 16 // advance X ptr by 4 floats + s_add_u32 s17, s17, 4 + s_branch .Ldot4_loop +.Ldot4_done: + + // Handle remaining 1-3 elements (scalar) +.Ldot1_loop: + s_cmp_ge_u32 s17, s14 + s_cbranch_scc1 .Ldot1_done + global_load_b32 v12, v5, s[4:5] + global_load_b32 v13, v6, s[8:9] + s_waitcnt vmcnt(0) + v_fmac_f32 v21, v12, v13 + v_add_nc_u32 v5, v5, 4 + v_add_nc_u32 v6, v6, 4 + s_add_u32 s17, s17, 1 + s_branch .Ldot1_loop +.Ldot1_done: + + // Y[gid] = acc + bias + s_waitcnt lgkmcnt(0) + v_add_f32 v21, v21, v20 + + // Store + v_lshlrev_b32 v1, 2, v1 + global_store_b32 v1, v21, s[10:11] + +.Lexit: + s_waitcnt vmcnt(0) + s_endpgm + +// Kernel descriptor +.rodata +.p2align 6 +.amdhsa_kernel superlinear_fwd + .amdhsa_group_segment_fixed_size 0 + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 48 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_next_free_vgpr 22 + .amdhsa_next_free_sgpr 19 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel + +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: superlinear_fwd + .symbol: superlinear_fwd.kd + .kernarg_segment_size: 48 + .group_segment_fixed_size: 0 + .private_segment_fixed_size: 0 + .kernarg_segment_align: 8 + .wavefront_size: 32 + .sgpr_count: 19 + .vgpr_count: 22 + .max_flat_workgroup_size: 256 + .args: + - { .size: 8, .offset: 0, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 8, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 16, .value_kind: global_buffer, .address_space: global } + - { .size: 8, .offset: 24, .value_kind: global_buffer, .address_space: global } + - { .size: 4, .offset: 32, .value_kind: by_value } + - { .size: 4, .offset: 36, .value_kind: by_value } + - { .size: 4, .offset: 40, .value_kind: by_value } +... +.end_amdgpu_metadata diff --git a/src/kernels/test_store.co b/src/kernels/test_store.co new file mode 100755 index 0000000..7fe22ef Binary files /dev/null and b/src/kernels/test_store.co differ diff --git a/src/kernels/test_store.s b/src/kernels/test_store.s new file mode 100644 index 0000000..5cdfefc --- /dev/null +++ b/src/kernels/test_store.s @@ -0,0 +1,41 @@ +// absolute minimal test: store 42.0 to y[workitem_id] +// kernargs: y pointer (u64) at offset 0 + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1102" +.amdhsa_code_object_version 5 + +.text +.globl test_store +.p2align 8 +.type test_store, @function +test_store: + // s[0:1] = kernarg pointer + s_load_b64 s[2:3], s[0:1], 0x00 // y ptr + s_waitcnt lgkmcnt(0) + + // build full 64-bit address in v[2:3] = s[2:3] + v0*4 + v_lshlrev_b32 v1, 2, v0 // v1 = workitem_id * 4 + v_add_co_u32 v2, vcc_lo, s2, v1 // v2 = y_lo + offset + v_add_co_ci_u32 v3, vcc_lo, s3, 0, vcc_lo // v3 = y_hi + carry + + // store constant 42.0 + v_mov_b32 v4, 0x42280000 // v4 = 42.0f + flat_store_b32 v[2:3], v4 + s_waitcnt vmcnt(0) lgkmcnt(0) + s_endpgm + +.rodata +.p2align 6 +.amdhsa_kernel test_store + .amdhsa_group_segment_fixed_size 0 + .amdhsa_private_segment_fixed_size 0 + .amdhsa_kernarg_size 8 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_next_free_vgpr 5 + .amdhsa_next_free_sgpr 4 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_wavefront_size32 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_ieee_mode 1 +.end_amdhsa_kernel diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..8d61cb3 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,25 @@ +//! KFD — Direct GPU compute via /dev/kfd for AMD RDNA3. +//! +//! No ROCm. No OpenCL. No HIP. Just ioctls. +//! +//! Provides: +//! - Device discovery and property queries +//! - VRAM/GTT/Userptr memory allocation with shared address space +//! - PM4 compute queue submission +//! - Kernel code object loading and dispatch +//! - Hand-written RDNA3 ASM kernels: matmul (3100 GFLOP/s), matvec, superlinear +//! +//! Usage: +//! let dev = HsaDevice::open()?; +//! let buf = dev.alloc_vram(1024)?; +//! dev.upload_f32(&buf, &data); +//! dev.dispatch_matmul(&a, &b, &c, M, N, K)?; + +pub mod ioctl; +pub mod memory; +pub mod queue; +pub mod dispatch; +pub mod compute; +mod device; + +pub use device::*; diff --git a/src/memory.rs b/src/memory.rs new file mode 100644 index 0000000..f8be1b8 --- /dev/null +++ b/src/memory.rs @@ -0,0 +1,328 @@ +//! GPU memory management via KFD ioctls. +//! +//! Three allocation types matching tinygrad's KFDIface.alloc(): +//! - VRAM: device-local, for weights and compute buffers +//! - GTT: system memory visible to GPU, for ring buffers and signals +//! - Userptr: user-allocated memory mapped to GPU + +use crate::ioctl; +use std::os::unix::io::RawFd; +use std::ptr; + +const MAP_NORESERVE: libc::c_int = 0x4000; + +/// A GPU-visible memory buffer with optional CPU-mapped view. +/// +/// Safety invariants: +/// - `va_addr` is always a valid GPU virtual address while the buffer lives +/// - `cpu_ptr` is non-null IFF `has_cpu_access` is true +/// - `mmap_addr` is the original mmap reservation (may differ from cpu_ptr for private VRAM) +/// - `kfd_fd` must remain open for the lifetime of this buffer (enforced by Arc in GpuAllocator) +pub struct GpuBuffer { + pub va_addr: u64, + pub size: u64, + pub handle: u64, + pub cpu_ptr: *mut u8, + /// Original mmap address for munmap on drop (may be PROT_NONE reservation for private VRAM) + mmap_addr: *mut libc::c_void, + mmap_size: usize, + pub kfd_fd: RawFd, + pub gpu_id: u32, + flags: u32, + has_cpu_access: bool, +} + +unsafe impl Send for GpuBuffer {} +unsafe impl Sync for GpuBuffer {} + +impl GpuBuffer { + /// True if this buffer can be read/written from CPU. + pub fn has_cpu_access(&self) -> bool { self.has_cpu_access } + + /// Panic message for null access. + fn assert_cpu_access(&self) { + assert!(self.has_cpu_access && !self.cpu_ptr.is_null(), + "attempted CPU access on GPU-only buffer (flags=0x{:08x})", self.flags); + } + + /// Write data to the buffer at the given byte offset. + pub fn write(&self, offset: usize, data: &[u8]) { + self.assert_cpu_access(); + assert!(offset + data.len() <= self.size as usize, + "write out of bounds: offset={} len={} size={}", offset, data.len(), self.size); + unsafe { + ptr::copy_nonoverlapping(data.as_ptr(), self.cpu_ptr.add(offset), data.len()); + } + } + + /// Read data from the buffer. + pub fn read(&self, offset: usize, len: usize) -> Vec { + self.assert_cpu_access(); + assert!(offset + len <= self.size as usize, + "read out of bounds: offset={} len={} size={}", offset, len, self.size); + let mut out = vec![0u8; len]; + unsafe { + ptr::copy_nonoverlapping(self.cpu_ptr.add(offset), out.as_mut_ptr(), len); + } + out + } + + /// Get a typed slice view of the buffer (requires CPU access). + pub fn as_slice(&self) -> &[T] { + self.assert_cpu_access(); + let count = self.size as usize / std::mem::size_of::(); + unsafe { std::slice::from_raw_parts(self.cpu_ptr as *const T, count) } + } + + /// Get a mutable typed slice view (requires &mut self to prevent aliasing). + pub fn as_slice_mut(&mut self) -> &mut [T] { + self.assert_cpu_access(); + let count = self.size as usize / std::mem::size_of::(); + unsafe { std::slice::from_raw_parts_mut(self.cpu_ptr as *mut T, count) } + } + + /// Write f32 slice into the buffer at byte offset. + pub fn write_f32(&self, offset: usize, data: &[f32]) { + let bytes = unsafe { + std::slice::from_raw_parts(data.as_ptr() as *const u8, data.len() * 4) + }; + self.write(offset, bytes); + } + + /// Read f32 slice from the buffer. + pub fn read_f32(&self, offset: usize, count: usize) -> Vec { + self.assert_cpu_access(); + assert!(offset + count * 4 <= self.size as usize, + "read_f32 out of bounds: offset={} count={} size={}", offset, count, self.size); + let mut out = vec![0.0f32; count]; + unsafe { + ptr::copy_nonoverlapping( + self.cpu_ptr.add(offset), + out.as_mut_ptr() as *mut u8, + count * 4, + ); + } + out + } +} + +impl Drop for GpuBuffer { + fn drop(&mut self) { + // Release GPU mapping and handle + if self.handle != 0 { + let gpu_ids = [self.gpu_id]; + let _ = ioctl::unmap_memory(self.kfd_fd, self.handle, &gpu_ids); + let _ = ioctl::free_memory(self.kfd_fd, self.handle); + } + // Release CPU mmap (covers both CPU-mapped and VA-reservation-only cases) + if !self.mmap_addr.is_null() && self.mmap_size > 0 { + unsafe { libc::munmap(self.mmap_addr, self.mmap_size); } + } + } +} + +/// GPU memory allocator. +pub struct GpuAllocator { + pub kfd_fd: RawFd, + pub drm_fd: RawFd, + pub gpu_id: u32, +} + +impl GpuAllocator { + /// Allocate VRAM (device-local). CPU-visible through resizable BAR. + pub fn alloc_vram(&self, size: u64) -> std::io::Result { + let flags = ioctl::ALLOC_MEM_FLAGS_VRAM + | ioctl::ALLOC_MEM_FLAGS_WRITABLE + | ioctl::ALLOC_MEM_FLAGS_EXECUTABLE + | ioctl::ALLOC_MEM_FLAGS_PUBLIC + | ioctl::ALLOC_MEM_FLAGS_NO_SUBSTITUTE; + self.alloc_internal(size, flags, false) + } + + /// Allocate GTT (system memory visible to GPU). Uncached, coherent. + /// Used for event pages, signal memory, control structures. + pub fn alloc_gtt(&self, size: u64) -> std::io::Result { + let flags = ioctl::ALLOC_MEM_FLAGS_GTT + | ioctl::ALLOC_MEM_FLAGS_WRITABLE + | ioctl::ALLOC_MEM_FLAGS_EXECUTABLE + | ioctl::ALLOC_MEM_FLAGS_NO_SUBSTITUTE + | ioctl::ALLOC_MEM_FLAGS_COHERENT + | ioctl::ALLOC_MEM_FLAGS_UNCACHED; + self.alloc_internal(size, flags, false) + } + + /// Allocate GTT with PUBLIC flag (CPU + GPU visible). + /// Used for ring buffers and read/write pointers. + pub fn alloc_gtt_public(&self, size: u64) -> std::io::Result { + let flags = ioctl::ALLOC_MEM_FLAGS_GTT + | ioctl::ALLOC_MEM_FLAGS_WRITABLE + | ioctl::ALLOC_MEM_FLAGS_EXECUTABLE + | ioctl::ALLOC_MEM_FLAGS_PUBLIC + | ioctl::ALLOC_MEM_FLAGS_NO_SUBSTITUTE + | ioctl::ALLOC_MEM_FLAGS_COHERENT + | ioctl::ALLOC_MEM_FLAGS_UNCACHED; + self.alloc_internal(size, flags, false) + } + + /// Allocate VRAM without PUBLIC flag (not CPU-accessible). + /// For internal GPU buffers like EOP and ctx_save_restore. + pub fn alloc_vram_private(&self, size: u64) -> std::io::Result { + let flags = ioctl::ALLOC_MEM_FLAGS_VRAM + | ioctl::ALLOC_MEM_FLAGS_WRITABLE + | ioctl::ALLOC_MEM_FLAGS_EXECUTABLE + | ioctl::ALLOC_MEM_FLAGS_NO_SUBSTITUTE; + + // Reserve VA space (no CPU access needed) + let addr = unsafe { + libc::mmap( + ptr::null_mut(), + size as usize, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0, + ) + }; + if addr == libc::MAP_FAILED { + return Err(std::io::Error::last_os_error()); + } + + #[cfg(debug_assertions)] + eprintln!(" vram_private: size=0x{:x} flags=0x{:08x} va=0x{:x}", size, flags, addr as u64); + let mem = ioctl::alloc_memory( + self.kfd_fd, addr as u64, size, self.gpu_id, flags, 0, + )?; + #[cfg(debug_assertions)] + eprintln!(" vram_private: handle=0x{:x} va=0x{:x}", mem.handle, mem.va_addr); + + // No CPU mmap for private VRAM — GPU-only buffer + let gpu_ids = [self.gpu_id]; + ioctl::map_memory(self.kfd_fd, mem.handle, &gpu_ids)?; + + Ok(GpuBuffer { + va_addr: mem.va_addr, + size: mem.size, + handle: mem.handle, + cpu_ptr: ptr::null_mut(), + mmap_addr: addr as *mut libc::c_void, + mmap_size: size as usize, + kfd_fd: self.kfd_fd, + gpu_id: self.gpu_id, + flags, + has_cpu_access: false, + }) + } + + /// Allocate userptr (user-managed memory mapped to GPU). + /// With COHERENT + UNCACHED — for ring buffers and control structures. + pub fn alloc_userptr(&self, size: u64) -> std::io::Result { + self.alloc_userptr_flags(size, ioctl::ALLOC_MEM_FLAGS_USERPTR + | ioctl::ALLOC_MEM_FLAGS_WRITABLE + | ioctl::ALLOC_MEM_FLAGS_EXECUTABLE + | ioctl::ALLOC_MEM_FLAGS_NO_SUBSTITUTE + | ioctl::ALLOC_MEM_FLAGS_COHERENT + | ioctl::ALLOC_MEM_FLAGS_UNCACHED) + } + + /// Allocate userptr with PUBLIC flag — for signal/scratch buffers. + /// Matches tinygrad's alloc(cpu_access=True) = flags 0xF0000004. + pub fn alloc_userptr_public(&self, size: u64) -> std::io::Result { + self.alloc_userptr_flags(size, ioctl::ALLOC_MEM_FLAGS_USERPTR + | ioctl::ALLOC_MEM_FLAGS_WRITABLE + | ioctl::ALLOC_MEM_FLAGS_EXECUTABLE + | ioctl::ALLOC_MEM_FLAGS_PUBLIC + | ioctl::ALLOC_MEM_FLAGS_NO_SUBSTITUTE) + } + + fn alloc_userptr_flags(&self, size: u64, flags: u32) -> std::io::Result { + + // Userptr: mmap first, then tell KFD about it + let addr = unsafe { + libc::mmap( + ptr::null_mut(), + size as usize, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED | libc::MAP_ANONYMOUS, + -1, 0, + ) + }; + if addr == libc::MAP_FAILED { + return Err(std::io::Error::last_os_error()); + } + + let mem = ioctl::alloc_memory( + self.kfd_fd, addr as u64, size, self.gpu_id, flags, addr as u64, + )?; + + let gpu_ids = [self.gpu_id]; + ioctl::map_memory(self.kfd_fd, mem.handle, &gpu_ids)?; + + Ok(GpuBuffer { + va_addr: mem.va_addr, + size: mem.size, + handle: mem.handle, + cpu_ptr: addr as *mut u8, + mmap_addr: addr as *mut libc::c_void, + mmap_size: size as usize, + kfd_fd: self.kfd_fd, + gpu_id: self.gpu_id, + flags, + has_cpu_access: true, + }) + } + + fn alloc_internal(&self, size: u64, flags: u32, _is_userptr: bool) -> std::io::Result { + #[cfg(debug_assertions)] + eprintln!(" alloc_internal: size=0x{:x} flags=0x{:08x}", size, flags); + // Reserve VA space + let addr = unsafe { + libc::mmap( + ptr::null_mut(), + size as usize, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0, + ) + }; + if addr == libc::MAP_FAILED { + return Err(std::io::Error::last_os_error()); + } + + // KFD alloc + let mem = ioctl::alloc_memory( + self.kfd_fd, addr as u64, size, self.gpu_id, flags, 0, + )?; + + // mmap the allocation for CPU access + let cpu_ptr = unsafe { + libc::mmap( + mem.va_addr as *mut libc::c_void, + mem.size as usize, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED | libc::MAP_FIXED, + self.drm_fd, + mem.mmap_offset as libc::off_t, + ) + }; + if cpu_ptr == libc::MAP_FAILED { + ioctl::free_memory(self.kfd_fd, mem.handle)?; + return Err(std::io::Error::last_os_error()); + } + + // Map to GPU + let gpu_ids = [self.gpu_id]; + ioctl::map_memory(self.kfd_fd, mem.handle, &gpu_ids)?; + + Ok(GpuBuffer { + va_addr: mem.va_addr, + size: mem.size, + handle: mem.handle, + cpu_ptr: cpu_ptr as *mut u8, + mmap_addr: cpu_ptr as *mut libc::c_void, + mmap_size: mem.size as usize, + kfd_fd: self.kfd_fd, + gpu_id: self.gpu_id, + flags, + has_cpu_access: true, + }) + } +} diff --git a/src/queue.rs b/src/queue.rs new file mode 100644 index 0000000..c0de508 --- /dev/null +++ b/src/queue.rs @@ -0,0 +1,398 @@ +//! Compute queue management: ring buffer, doorbell, PM4 submission. +//! +//! PM4 (Packet Manager 4) is the command packet format for GFX/compute. +//! We use QUEUE_TYPE_COMPUTE (raw PM4) rather than AQL for simplicity. + +use crate::ioctl; +use crate::memory::{GpuAllocator, GpuBuffer}; +use std::os::unix::io::RawFd; +use std::ptr; + +/// PM4 packet header: type 3 +const fn pkt3(opcode: u32, count: u32) -> u32 { + (3 << 30) | ((count.wrapping_sub(1) & 0x3FFF) << 16) | (opcode << 8) +} + +// ─── PM4 opcodes (GFX11/RDNA3) ───────────────────────────── + +pub const PACKET3_NOP: u32 = 0x10; +pub const PACKET3_SET_SH_REG: u32 = 0x76; +pub const PACKET3_DISPATCH_DIRECT: u32 = 0x15; +pub const PACKET3_ACQUIRE_MEM: u32 = 0x58; +pub const PACKET3_RELEASE_MEM: u32 = 0x49; +pub const PACKET3_WAIT_REG_MEM: u32 = 0x3C; +pub const PACKET3_EVENT_WRITE: u32 = 0x46; + +// SET_SH_REG base for GFX11 +pub const SH_REG_BASE: u32 = 0x2C00; + +// Compute shader registers (offsets from SH_REG_BASE) +// These are GFX11 (RDNA3) specific — from gc_11_0_0_offset.h +pub const COMPUTE_PGM_LO: u32 = 0x2E0C - SH_REG_BASE; +pub const COMPUTE_PGM_HI: u32 = 0x2E0D - SH_REG_BASE; +pub const COMPUTE_PGM_RSRC1: u32 = 0x2E12 - SH_REG_BASE; +pub const COMPUTE_PGM_RSRC2: u32 = 0x2E13 - SH_REG_BASE; +pub const COMPUTE_PGM_RSRC3: u32 = 0x2E28 - SH_REG_BASE; +pub const COMPUTE_TMPRING_SIZE: u32 = 0x2E18 - SH_REG_BASE; +pub const COMPUTE_USER_DATA_0: u32 = 0x2E40 - SH_REG_BASE; +pub const COMPUTE_RESOURCE_LIMITS: u32 = 0x2E15 - SH_REG_BASE; +pub const COMPUTE_START_X: u32 = 0x2E04 - SH_REG_BASE; +pub const COMPUTE_NUM_THREAD_X: u32 = 0x2E07 - SH_REG_BASE; +pub const COMPUTE_RESTART_X: u32 = 0x2E1B - SH_REG_BASE; + +// DISPATCH_INITIATOR bits +pub const COMPUTE_SHADER_EN: u32 = 1 << 0; +pub const CS_W32_EN: u32 = 1 << 15; // wave32 mode (RDNA3) +pub const FORCE_START_AT_000: u32 = 1 << 2; + +// ACQUIRE_MEM / RELEASE_MEM cache control (GFX11) +pub const GCR_GLI_INV_GL1: u32 = 1 << 0; +pub const GCR_GL2_INV: u32 = 1 << 14; +pub const GCR_GL2_WB: u32 = 1 << 15; +pub const GCR_GLM_INV: u32 = 1 << 5; +pub const GCR_GLM_WB: u32 = 1 << 4; +pub const GCR_GLV_INV: u32 = 1 << 9; +pub const GCR_GLK_INV: u32 = 1 << 12; +pub const GCR_SEQ_FORWARD: u32 = 1 << 16; + +// RELEASE_MEM event types +pub const EVENT_TYPE_CACHE_FLUSH: u32 = 0x06; +pub const EVENT_TYPE_CS_PARTIAL_FLUSH: u32 = 0x04; +pub const DATA_SEL_VALUE_32BIT: u32 = 1; +pub const DATA_SEL_VALUE_64BIT: u32 = 2; +pub const INT_SEL_SEND_DATA_AFTER_WR_CONFIRM: u32 = 3; + +// Wait conditions +pub const WAIT_REG_MEM_FUNCTION_GEQ: u32 = 5; +pub const WAIT_REG_MEM_MEM_SPACE_MEM: u32 = 1; + +/// Ring buffer size (16MB, matching tinygrad production) +const RING_SIZE: u64 = 0x1000000; + +/// Size for EOP (end-of-pipe) buffer +const EOP_SIZE: u64 = 0x1000; + +/// ctx_save_restore size (ioctl field value, from sysfs cwsr_size) +const CTX_SAVE_SIZE: u64 = 0xAA4000; + +/// Debug memory size: wave_count * 32, rounded to 64. +/// wave_count = max_waves_per_simd * simd_count = 16 * 64 = 1024 +/// 1024 * 32 = 32768 = 0x8000 +const DEBUG_MEM_SIZE: u64 = 0x8000; + +/// Actual buffer allocation = cwsr_size + debug_memory, page-aligned. +/// Kernel validates the BO size matches this exactly (since 6.11). +const CTX_SAVE_ALLOC_SIZE: u64 = (CTX_SAVE_SIZE + DEBUG_MEM_SIZE + 0xFFF) & !0xFFF; // 0xAAC000 + +/// ctl_stack_size +const CTL_STACK_SIZE: u32 = 0x4000; + +/// A PM4 compute queue on the GPU. +pub struct ComputeQueue { + pub queue_id: u32, + pub ring: GpuBuffer, + pub rw_ptrs: GpuBuffer, + doorbell_ptr: *mut u64, + _doorbell_mmap: *mut libc::c_void, + _doorbell_size: usize, + _eop: GpuBuffer, + _ctx_save: GpuBuffer, + pub put: u64, + kfd_fd: RawFd, +} + +unsafe impl Send for ComputeQueue {} +unsafe impl Sync for ComputeQueue {} + +impl ComputeQueue { + /// Create a new PM4 compute queue. + pub fn new(alloc: &GpuAllocator) -> std::io::Result { + // Ring buffer (GTT+PUBLIC, flags 0xF6000002 matching tinygrad) + let ring = alloc.alloc_gtt_public(RING_SIZE) + .map_err(|e| { eprintln!(" ring alloc failed: {}", e); e })?; + + // Read/write pointer memory (GTT+PUBLIC, flags 0xF6000002) + let rw_ptrs = alloc.alloc_gtt_public(0x100) + .map_err(|e| { eprintln!(" rw_ptrs alloc failed: {}", e); e })?; + + // Zero the write/read pointers + unsafe { + ptr::write_bytes(rw_ptrs.cpu_ptr, 0, rw_ptrs.size as usize); + } + + // EOP (VRAM private, flags 0xD0000001 matching tinygrad) + let eop = alloc.alloc_vram_private(EOP_SIZE) + .map_err(|e| { eprintln!(" eop alloc failed: {}", e); e })?; + + // ctx_save_restore: alloc CTX_SAVE_ALLOC_SIZE (cwsr + debug), ioctl field is CTX_SAVE_SIZE + // VRAM private, flags 0xD0000001 matching tinygrad + let ctx_save = alloc.alloc_vram_private(CTX_SAVE_ALLOC_SIZE) + .map_err(|e| { eprintln!(" ctx_save alloc failed: {}", e); e })?; + + // Create queue via KFD + let mut args = ioctl::CreateQueueArgs { + ring_base_address: ring.va_addr, + ring_size: RING_SIZE as u32, + gpu_id: alloc.gpu_id, + queue_type: ioctl::QUEUE_TYPE_COMPUTE, + queue_percentage: ioctl::MAX_QUEUE_PERCENTAGE, + queue_priority: 7, + write_pointer_address: rw_ptrs.va_addr + 0x38, + read_pointer_address: rw_ptrs.va_addr + 0x80, + eop_buffer_address: eop.va_addr, + eop_buffer_size: EOP_SIZE, + ctx_save_restore_address: ctx_save.va_addr, + ctx_save_restore_size: CTX_SAVE_SIZE as u32, + ctl_stack_size: CTL_STACK_SIZE, + ..Default::default() + }; + ioctl::create_queue(alloc.kfd_fd, &mut args)?; + + // mmap doorbell + let doorbell_offset = args.doorbell_offset; + let doorbell_page = doorbell_offset & !0x1FFF; + let doorbell_off_in_page = (doorbell_offset & 0x1FFF) as usize; + let doorbell_size = 8192; + let doorbell_mmap = unsafe { + libc::mmap( + ptr::null_mut(), + doorbell_size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + alloc.kfd_fd, + doorbell_page as libc::off_t, + ) + }; + if doorbell_mmap == libc::MAP_FAILED { + return Err(std::io::Error::last_os_error()); + } + let doorbell_ptr = unsafe { + (doorbell_mmap as *mut u8).add(doorbell_off_in_page) as *mut u64 + }; + + Ok(ComputeQueue { + queue_id: args.queue_id, + ring, + rw_ptrs, + doorbell_ptr, + _doorbell_mmap: doorbell_mmap, + _doorbell_size: doorbell_size, + _eop: eop, + _ctx_save: ctx_save, + put: 0, + kfd_fd: alloc.kfd_fd, + }) + } + + /// Check ring buffer space. Panics if GPU hasn't consumed enough packets. + fn check_ring_space(&self, dwords_needed: u64) { + let ring_dwords = self.ring.size as u64 / 4; + let read_ptr = unsafe { + (self.rw_ptrs.cpu_ptr.add(0x80) as *const u64).read_volatile() + }; + let used = self.put.wrapping_sub(read_ptr); + assert!(used + dwords_needed < ring_dwords, + "ring buffer overflow: put={} read={} need={} capacity={}", + self.put, read_ptr, dwords_needed, ring_dwords); + } + + /// Write a DWORD to the ring and advance. + #[inline] + fn q(&mut self, val: u32) { + let ring_dwords = self.ring.size as u64 / 4; + let offset = (self.put % ring_dwords) as usize; + unsafe { + let ptr = self.ring.cpu_ptr as *mut u32; + ptr.add(offset).write_volatile(val); + } + self.put += 1; + } + + /// Write a PM4 type 3 packet. + fn pkt3(&mut self, opcode: u32, vals: &[u32]) { + self.check_ring_space(1 + vals.len() as u64); + self.q(pkt3(opcode, vals.len() as u32)); + for &v in vals { + self.q(v); + } + } + + /// Write SET_SH_REG packet (set one or more shader registers). + pub fn set_sh_reg(&mut self, offset: u32, values: &[u32]) { + let mut data = vec![offset]; + data.extend_from_slice(values); + self.pkt3(PACKET3_SET_SH_REG, &data); + } + + /// Memory barrier before kernel dispatch. + /// 7 data dwords on gfx11 (not 6!) — includes GCR_CNTL as 7th word. + pub fn acquire_mem(&mut self) { + self.pkt3(PACKET3_ACQUIRE_MEM, &[ + 0, // CP_COHER_CNTL + 0xFFFFFFFF, // COHER_SIZE lo + 0xFFFFFFFF, // COHER_SIZE hi + 0, // COHER_BASE lo + 0, // COHER_BASE hi + 0, // POLL_INTERVAL + 0x000003F0, // GCR_CNTL (gfx11 requires this 7th dword) + ]); + } + + /// Write a completion signal + send interrupt to KFD. + /// Must send TWO RELEASE_MEM packets (matching tinygrad): + /// 1. Write signal value to signal_addr (with cache flush) + /// 2. Write event_id to event mailbox (interrupt notification) + /// Without the second packet, MES hangs on queue destruction. + pub fn signal(&mut self, signal_addr: u64, signal_value: u32, + event_mailbox_ptr: u64, event_id: u32) { + // Packet 1: write signal value WITH cache flush, NO interrupt + self.pkt3(PACKET3_RELEASE_MEM, &[ + 0x0070f514, // event_type=20, event_index=5, GCR cache flush + 0x20000000, // data_sel=1(32bit), int_sel=0(none) + signal_addr as u32, + (signal_addr >> 32) as u32, + signal_value, + 0, + 0, + ]); + + // Packet 2: cache flush + event mailbox + interrupt + // by the time this executes, packet 1's signal write is done + self.pkt3(PACKET3_RELEASE_MEM, &[ + 0x0070f514, // event_type=20, event_index=5, GCR cache flush + 0x22000000, // data_sel=1(32bit), int_sel=2(interrupt after write) + event_mailbox_ptr as u32, + (event_mailbox_ptr >> 32) as u32, + event_id, + 0, + event_id, // ctxid = event_id + ]); + } + + /// Wait until a memory address contains >= expected value. + pub fn wait_reg_mem(&mut self, addr: u64, expected: u32) { + self.pkt3(PACKET3_WAIT_REG_MEM, &[ + (WAIT_REG_MEM_FUNCTION_GEQ) | (WAIT_REG_MEM_MEM_SPACE_MEM << 4), + addr as u32, + (addr >> 32) as u32, + expected, + 0xFFFFFFFF, // mask + 4, // poll interval + ]); + } + + /// Dispatch a compute kernel. + /// + /// `pgm_addr`: GPU virtual address of kernel code + /// `rsrc1`, `rsrc2`, `rsrc3`: from kernel descriptor + /// `kernargs_addr`: GPU virtual address of kernel arguments buffer + /// `grid`: (global_x, global_y, global_z) in workitems + /// `block`: (local_x, local_y, local_z) workgroup size + pub fn dispatch(&mut self, + pgm_addr: u64, rsrc1: u32, rsrc2: u32, rsrc3: u32, + kernargs_addr: u64, scratch_addr: u64, + grid: [u32; 3], block: [u32; 3]) { + self.dispatch_lds(pgm_addr, rsrc1, rsrc2, rsrc3, kernargs_addr, scratch_addr, grid, block, 0); + } + + /// Dispatch with explicit LDS allocation (group_segment_fixed_size in bytes). + pub fn dispatch_lds(&mut self, + pgm_addr: u64, rsrc1: u32, rsrc2: u32, rsrc3: u32, + kernargs_addr: u64, scratch_addr: u64, + grid: [u32; 3], block: [u32; 3], + lds_bytes: u32) { + // Cache invalidate + self.acquire_mem(); + + // Program address (shifted right by 8 per AMD convention) + self.set_sh_reg(COMPUTE_PGM_LO, &[ + (pgm_addr >> 8) as u32, + (pgm_addr >> 40) as u32, + ]); + + // Resource descriptors — set PRIV bit (1<<20) on rsrc1 for GFX11 (cwsr workaround) + let rsrc1 = rsrc1 | (1 << 20); + // Patch rsrc2 with LDS_SIZE (bits 15:23) in 128-dword (512-byte) granularity + let lds_alloc = (lds_bytes + 511) / 512; + let rsrc2 = (rsrc2 & !(0x1FF << 15)) | (lds_alloc << 15); + self.set_sh_reg(COMPUTE_PGM_RSRC1, &[rsrc1, rsrc2]); + self.set_sh_reg(COMPUTE_PGM_RSRC3, &[rsrc3]); + + // scratch ring: WAVESIZE=0 for kernels with no private segment + // setting WAVESIZE>0 limits concurrent waves and causes hangs at 32+ WGs + self.set_sh_reg(COMPUTE_TMPRING_SIZE, &[0x00000000]); + + // Scratch base address (required on gfx11 with has_scratch_base_registers) + // Register 0x0210 = COMPUTE_DISPATCH_SCRATCH_BASE_LO/HI + self.set_sh_reg(0x0210, &[ + (scratch_addr >> 8) as u32, + (scratch_addr >> 40) as u32, + ]); + + // Restart counters + self.set_sh_reg(COMPUTE_RESTART_X, &[0, 0, 0]); + + // Kernel arguments pointer + self.set_sh_reg(COMPUTE_USER_DATA_0, &[ + kernargs_addr as u32, + (kernargs_addr >> 32) as u32, + ]); + + // Resource limits: 0 = no limit. The hardware manages per-CU resource + // allocation and won't launch a WG unless all its waves can be allocated. + // Barrier deadlocks only happen with manual limits that are too low. + self.set_sh_reg(COMPUTE_RESOURCE_LIMITS, &[0]); + + // Start offsets + workgroup size (contiguous regs 0x204-0x20C) + // Matches tinygrad: start_x/y/z, local_x/y/z, 0, 0 + self.set_sh_reg(COMPUTE_START_X, &[ + 0, 0, 0, // start_x, start_y, start_z + block[0], block[1], block[2], // num_thread_x, num_thread_y, num_thread_z + 0, 0, // padding (perfcount_enable, etc.) + ]); + + // DISPATCH_DIRECT: grid dimensions + initiator + self.pkt3(PACKET3_DISPATCH_DIRECT, &[ + grid[0], grid[1], grid[2], + COMPUTE_SHADER_EN | CS_W32_EN | FORCE_START_AT_000, + ]); + + // CS_PARTIAL_FLUSH after dispatch (required, matches tinygrad) + self.pkt3(PACKET3_EVENT_WRITE, &[0x0407]); + } + + /// Submit all queued packets to the GPU by ringing the doorbell. + pub fn submit(&mut self) { + // Memory fence to ensure ring writes are visible + std::sync::atomic::fence(std::sync::atomic::Ordering::Release); + + // Update write pointer + // Write pointer is at offset +0x38 in rw_ptrs buffer + let wp = unsafe { self.rw_ptrs.cpu_ptr.add(0x38) as *mut u64 }; + unsafe { wp.write_volatile(self.put); } + + // Ring doorbell (u64 write, same as tinygrad) + std::sync::atomic::fence(std::sync::atomic::Ordering::Release); + unsafe { self.doorbell_ptr.write_volatile(self.put); } + } + + /// Spin-wait until a 32-bit signal address contains the expected value. + pub fn poll_signal(signal_ptr: *const u32, expected: u32, timeout_us: u64) -> bool { + let start = std::time::Instant::now(); + loop { + let val = unsafe { signal_ptr.read_volatile() }; + if val >= expected { return true; } + if start.elapsed().as_micros() as u64 > timeout_us { return false; } + std::hint::spin_loop(); + } + } +} + +impl Drop for ComputeQueue { + fn drop(&mut self) { + let _ = ioctl::destroy_queue(self.kfd_fd, self.queue_id); + if !self._doorbell_mmap.is_null() { + unsafe { libc::munmap(self._doorbell_mmap, self._doorbell_size); } + } + } +} diff --git a/target/.rustc_info.json b/target/.rustc_info.json new file mode 100644 index 0000000..24546e7 --- /dev/null +++ b/target/.rustc_info.json @@ -0,0 +1 @@ +{"rustc_fingerprint":8771983278988161788,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.so\nlib___.so\nlib___.a\nlib___.so\n/home/alice/.rustup/toolchains/stable-x86_64-unknown-linux-gnu\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"gnu\"\ntarget_family=\"unix\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"linux\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"unknown\"\nunix\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.94.0 (4a4ef493e 2026-03-02)\nbinary: rustc\ncommit-hash: 4a4ef493e3a1488c6e321570238084b38948f6db\ncommit-date: 2026-03-02\nhost: x86_64-unknown-linux-gnu\nrelease: 1.94.0\nLLVM version: 21.1.8\n","stderr":""}},"successes":{}} \ No newline at end of file diff --git a/target/CACHEDIR.TAG b/target/CACHEDIR.TAG new file mode 100644 index 0000000..20d7c31 --- /dev/null +++ b/target/CACHEDIR.TAG @@ -0,0 +1,3 @@ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by cargo. +# For information about cache directory tags see https://bford.info/cachedir/ diff --git a/target/debug/.cargo-lock b/target/debug/.cargo-lock new file mode 100644 index 0000000..e69de29 diff --git a/target/debug/.fingerprint/kfd-2447e9870f0e8752/invoked.timestamp b/target/debug/.fingerprint/kfd-2447e9870f0e8752/invoked.timestamp new file mode 100644 index 0000000..e00328d --- /dev/null +++ b/target/debug/.fingerprint/kfd-2447e9870f0e8752/invoked.timestamp @@ -0,0 +1 @@ +This file has an mtime of when this was started. \ No newline at end of file diff --git a/target/debug/.fingerprint/kfd-2447e9870f0e8752/output-lib-kfd b/target/debug/.fingerprint/kfd-2447e9870f0e8752/output-lib-kfd new file mode 100644 index 0000000..47d13e6 --- /dev/null +++ b/target/debug/.fingerprint/kfd-2447e9870f0e8752/output-lib-kfd @@ -0,0 +1,50 @@ +{"$message_type":"diagnostic","message":"unused import: `KernelEntry`","code":{"code":"unused_imports","explanation":null},"level":"warning","spans":[{"file_name":"src/device.rs","byte_start":619,"byte_end":630,"line_start":17,"line_end":17,"column_start":57,"column_end":68,"is_primary":true,"text":[{"text":"use crate::dispatch::{CodeObject, GpuProgram, KernArgs, KernelEntry};","highlight_start":57,"highlight_end":68}],"label":null,"suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"`#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"remove the unused import","code":null,"level":"help","spans":[{"file_name":"src/device.rs","byte_start":617,"byte_end":630,"line_start":17,"line_end":17,"column_start":55,"column_end":68,"is_primary":true,"text":[{"text":"use crate::dispatch::{CodeObject, GpuProgram, KernArgs, KernelEntry};","highlight_start":55,"highlight_end":68}],"label":null,"suggested_replacement":"","suggestion_applicability":"MachineApplicable","expansion":null}],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[33mwarning\u001b[0m\u001b[1m: unused import: `KernelEntry`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:17:57\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m17\u001b[0m \u001b[1m\u001b[94m|\u001b[0m use crate::dispatch::{CodeObject, GpuProgram, KernArgs, KernelEntry};\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[33m^^^^^^^^^^^\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mnote\u001b[0m: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/ioctl.rs","byte_start":990,"byte_end":994,"line_start":26,"line_end":26,"column_start":15,"column_end":19,"is_primary":true,"text":[{"text":" let ret = libc::ioctl(fd, request as libc::c_ulong, arg as *mut T);","highlight_start":15,"highlight_end":19}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/ioctl.rs:26:15\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m26\u001b[0m \u001b[1m\u001b[94m|\u001b[0m let ret = libc::ioctl(fd, request as libc::c_ulong, arg as *mut T);\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":4408,"byte_end":4412,"line_start":119,"line_end":119,"column_start":22,"column_end":26,"is_primary":true,"text":[{"text":" unsafe { libc::munmap(self.mmap_addr, self.mmap_size); }","highlight_start":22,"highlight_end":26}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:119:22\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m119\u001b[0m \u001b[1m\u001b[94m|\u001b[0m unsafe { libc::munmap(self.mmap_addr, self.mmap_size); }\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":6674,"byte_end":6678,"line_start":177,"line_end":177,"column_start":13,"column_end":17,"is_primary":true,"text":[{"text":" libc::mmap(","highlight_start":13,"highlight_end":17}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:177:13\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m177\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::mmap(\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":6766,"byte_end":6770,"line_start":180,"line_end":180,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::PROT_NONE,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:180:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m180\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_NONE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":6799,"byte_end":6803,"line_start":181,"line_end":181,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:181:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m181\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":6819,"byte_end":6823,"line_start":181,"line_end":181,"column_start":37,"column_end":41,"is_primary":true,"text":[{"text":" libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,","highlight_start":37,"highlight_end":41}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:181:37\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m181\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":6923,"byte_end":6927,"line_start":185,"line_end":185,"column_start":20,"column_end":24,"is_primary":true,"text":[{"text":" if addr == libc::MAP_FAILED {","highlight_start":20,"highlight_end":24}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:185:20\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m185\u001b[0m \u001b[1m\u001b[94m|\u001b[0m if addr == libc::MAP_FAILED {\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":9145,"byte_end":9149,"line_start":240,"line_end":240,"column_start":13,"column_end":17,"is_primary":true,"text":[{"text":" libc::mmap(","highlight_start":13,"highlight_end":17}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:240:13\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m240\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::mmap(\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":9237,"byte_end":9241,"line_start":243,"line_end":243,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::PROT_READ | libc::PROT_WRITE,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:243:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m243\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_READ | libc::PROT_WRITE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":9255,"byte_end":9259,"line_start":243,"line_end":243,"column_start":35,"column_end":39,"is_primary":true,"text":[{"text":" libc::PROT_READ | libc::PROT_WRITE,","highlight_start":35,"highlight_end":39}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:243:35\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m243\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_READ | libc::PROT_WRITE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":9289,"byte_end":9293,"line_start":244,"line_end":244,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::MAP_SHARED | libc::MAP_ANONYMOUS,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:244:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m244\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_SHARED | libc::MAP_ANONYMOUS,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":9308,"byte_end":9312,"line_start":244,"line_end":244,"column_start":36,"column_end":40,"is_primary":true,"text":[{"text":" libc::MAP_SHARED | libc::MAP_ANONYMOUS,","highlight_start":36,"highlight_end":40}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:244:36\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m244\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_SHARED | libc::MAP_ANONYMOUS,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":9396,"byte_end":9400,"line_start":248,"line_end":248,"column_start":20,"column_end":24,"is_primary":true,"text":[{"text":" if addr == libc::MAP_FAILED {","highlight_start":20,"highlight_end":24}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:248:20\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m248\u001b[0m \u001b[1m\u001b[94m|\u001b[0m if addr == libc::MAP_FAILED {\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":10383,"byte_end":10387,"line_start":278,"line_end":278,"column_start":13,"column_end":17,"is_primary":true,"text":[{"text":" libc::mmap(","highlight_start":13,"highlight_end":17}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:278:13\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m278\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::mmap(\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":10475,"byte_end":10479,"line_start":281,"line_end":281,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::PROT_NONE,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:281:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m281\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_NONE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":10508,"byte_end":10512,"line_start":282,"line_end":282,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:282:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m282\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":10528,"byte_end":10532,"line_start":282,"line_end":282,"column_start":37,"column_end":41,"is_primary":true,"text":[{"text":" libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,","highlight_start":37,"highlight_end":41}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:282:37\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m282\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | MAP_NORESERVE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":10632,"byte_end":10636,"line_start":286,"line_end":286,"column_start":20,"column_end":24,"is_primary":true,"text":[{"text":" if addr == libc::MAP_FAILED {","highlight_start":20,"highlight_end":24}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:286:20\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m286\u001b[0m \u001b[1m\u001b[94m|\u001b[0m if addr == libc::MAP_FAILED {\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":10948,"byte_end":10952,"line_start":297,"line_end":297,"column_start":13,"column_end":17,"is_primary":true,"text":[{"text":" libc::mmap(","highlight_start":13,"highlight_end":17}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:297:13\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m297\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::mmap(\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":11061,"byte_end":11065,"line_start":300,"line_end":300,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::PROT_READ | libc::PROT_WRITE,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:300:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m300\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_READ | libc::PROT_WRITE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":11079,"byte_end":11083,"line_start":300,"line_end":300,"column_start":35,"column_end":39,"is_primary":true,"text":[{"text":" libc::PROT_READ | libc::PROT_WRITE,","highlight_start":35,"highlight_end":39}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:300:35\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m300\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_READ | libc::PROT_WRITE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":11113,"byte_end":11117,"line_start":301,"line_end":301,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::MAP_SHARED | libc::MAP_FIXED,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:301:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m301\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_SHARED | libc::MAP_FIXED,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":11132,"byte_end":11136,"line_start":301,"line_end":301,"column_start":36,"column_end":40,"is_primary":true,"text":[{"text":" libc::MAP_SHARED | libc::MAP_FIXED,","highlight_start":36,"highlight_end":40}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:301:36\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m301\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_SHARED | libc::MAP_FIXED,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":11273,"byte_end":11277,"line_start":306,"line_end":306,"column_start":23,"column_end":27,"is_primary":true,"text":[{"text":" if cpu_ptr == libc::MAP_FAILED {","highlight_start":23,"highlight_end":27}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:306:23\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m306\u001b[0m \u001b[1m\u001b[94m|\u001b[0m if cpu_ptr == libc::MAP_FAILED {\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":6048,"byte_end":6052,"line_start":157,"line_end":157,"column_start":13,"column_end":17,"is_primary":true,"text":[{"text":" libc::mmap(","highlight_start":13,"highlight_end":17}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:157:13\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m157\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::mmap(\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":6140,"byte_end":6144,"line_start":160,"line_end":160,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::PROT_READ | libc::PROT_WRITE,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:160:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m160\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_READ | libc::PROT_WRITE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":6158,"byte_end":6162,"line_start":160,"line_end":160,"column_start":35,"column_end":39,"is_primary":true,"text":[{"text":" libc::PROT_READ | libc::PROT_WRITE,","highlight_start":35,"highlight_end":39}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:160:35\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m160\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::PROT_READ | libc::PROT_WRITE,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":6192,"byte_end":6196,"line_start":161,"line_end":161,"column_start":17,"column_end":21,"is_primary":true,"text":[{"text":" libc::MAP_SHARED,","highlight_start":17,"highlight_end":21}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:161:17\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m161\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::MAP_SHARED,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":6339,"byte_end":6343,"line_start":166,"line_end":166,"column_start":29,"column_end":33,"is_primary":true,"text":[{"text":" if doorbell_mmap == libc::MAP_FAILED {","highlight_start":29,"highlight_end":33}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:166:29\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m166\u001b[0m \u001b[1m\u001b[94m|\u001b[0m if doorbell_mmap == libc::MAP_FAILED {\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":15460,"byte_end":15464,"line_start":395,"line_end":395,"column_start":22,"column_end":26,"is_primary":true,"text":[{"text":" unsafe { libc::munmap(self._doorbell_mmap, self._doorbell_size); }","highlight_start":22,"highlight_end":26}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:395:22\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m395\u001b[0m \u001b[1m\u001b[94m|\u001b[0m unsafe { libc::munmap(self._doorbell_mmap, self._doorbell_size); }\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/device.rs","byte_start":10449,"byte_end":10453,"line_start":272,"line_end":272,"column_start":31,"column_end":35,"is_primary":true,"text":[{"text":" let kfd_fd = unsafe { libc::open(b\"/dev/kfd\\0\".as_ptr() as _, libc::O_RDWR) };","highlight_start":31,"highlight_end":35}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:272:31\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m272\u001b[0m \u001b[1m\u001b[94m|\u001b[0m let kfd_fd = unsafe { libc::open(b\"/dev/kfd\\0\".as_ptr() as _, libc::O_RDWR) };\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/device.rs","byte_start":10489,"byte_end":10493,"line_start":272,"line_end":272,"column_start":71,"column_end":75,"is_primary":true,"text":[{"text":" let kfd_fd = unsafe { libc::open(b\"/dev/kfd\\0\".as_ptr() as _, libc::O_RDWR) };","highlight_start":71,"highlight_end":75}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:272:71\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m272\u001b[0m \u001b[1m\u001b[94m|\u001b[0m let kfd_fd = unsafe { libc::open(b\"/dev/kfd\\0\".as_ptr() as _, libc::O_RDWR) };\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/device.rs","byte_start":12207,"byte_end":12211,"line_start":312,"line_end":312,"column_start":31,"column_end":35,"is_primary":true,"text":[{"text":" let drm_fd = unsafe { libc::open(drm_path.as_ptr() as _, libc::O_RDWR) };","highlight_start":31,"highlight_end":35}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:312:31\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m312\u001b[0m \u001b[1m\u001b[94m|\u001b[0m let drm_fd = unsafe { libc::open(drm_path.as_ptr() as _, libc::O_RDWR) };\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/device.rs","byte_start":12242,"byte_end":12246,"line_start":312,"line_end":312,"column_start":66,"column_end":70,"is_primary":true,"text":[{"text":" let drm_fd = unsafe { libc::open(drm_path.as_ptr() as _, libc::O_RDWR) };","highlight_start":66,"highlight_end":70}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:312:66\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m312\u001b[0m \u001b[1m\u001b[94m|\u001b[0m let drm_fd = unsafe { libc::open(drm_path.as_ptr() as _, libc::O_RDWR) };\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/device.rs","byte_start":12304,"byte_end":12308,"line_start":314,"line_end":314,"column_start":22,"column_end":26,"is_primary":true,"text":[{"text":" unsafe { libc::close(kfd_fd); }","highlight_start":22,"highlight_end":26}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:314:22\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m314\u001b[0m \u001b[1m\u001b[94m|\u001b[0m unsafe { libc::close(kfd_fd); }\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/device.rs","byte_start":31466,"byte_end":31470,"line_start":778,"line_end":778,"column_start":13,"column_end":17,"is_primary":true,"text":[{"text":" libc::close(self.drm_fd);","highlight_start":13,"highlight_end":17}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:778:13\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m778\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::close(self.drm_fd);\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/device.rs","byte_start":31504,"byte_end":31508,"line_start":779,"line_end":779,"column_start":13,"column_end":17,"is_primary":true,"text":[{"text":" libc::close(self.kfd_fd);","highlight_start":13,"highlight_end":17}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:779:13\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m779\u001b[0m \u001b[1m\u001b[94m|\u001b[0m libc::close(self.kfd_fd);\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":7747,"byte_end":7751,"line_start":206,"line_end":206,"column_start":37,"column_end":41,"is_primary":true,"text":[{"text":" mmap_addr: addr as *mut libc::c_void,","highlight_start":37,"highlight_end":41}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:206:37\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m206\u001b[0m \u001b[1m\u001b[94m|\u001b[0m mmap_addr: addr as *mut libc::c_void,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/ioctl.rs","byte_start":1017,"byte_end":1021,"line_start":26,"line_end":26,"column_start":42,"column_end":46,"is_primary":true,"text":[{"text":" let ret = libc::ioctl(fd, request as libc::c_ulong, arg as *mut T);","highlight_start":42,"highlight_end":46}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/ioctl.rs:26:42\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m26\u001b[0m \u001b[1m\u001b[94m|\u001b[0m let ret = libc::ioctl(fd, request as libc::c_ulong, arg as *mut T);\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":381,"byte_end":385,"line_start":12,"line_end":12,"column_start":22,"column_end":26,"is_primary":true,"text":[{"text":"const MAP_NORESERVE: libc::c_int = 0x4000;","highlight_start":22,"highlight_end":26}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:12:22\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m12\u001b[0m \u001b[1m\u001b[94m|\u001b[0m const MAP_NORESERVE: libc::c_int = 0x4000;\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":1048,"byte_end":1052,"line_start":27,"line_end":27,"column_start":21,"column_end":25,"is_primary":true,"text":[{"text":" mmap_addr: *mut libc::c_void,","highlight_start":21,"highlight_end":25}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:27:21\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m27\u001b[0m \u001b[1m\u001b[94m|\u001b[0m mmap_addr: *mut libc::c_void,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":9904,"byte_end":9908,"line_start":264,"line_end":264,"column_start":37,"column_end":41,"is_primary":true,"text":[{"text":" mmap_addr: addr as *mut libc::c_void,","highlight_start":37,"highlight_end":41}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:264:37\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m264\u001b[0m \u001b[1m\u001b[94m|\u001b[0m mmap_addr: addr as *mut libc::c_void,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":11213,"byte_end":11217,"line_start":303,"line_end":303,"column_start":36,"column_end":40,"is_primary":true,"text":[{"text":" mem.mmap_offset as libc::off_t,","highlight_start":36,"highlight_end":40}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:303:36\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m303\u001b[0m \u001b[1m\u001b[94m|\u001b[0m mem.mmap_offset as libc::off_t,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":6273,"byte_end":6277,"line_start":163,"line_end":163,"column_start":34,"column_end":38,"is_primary":true,"text":[{"text":" doorbell_page as libc::off_t,","highlight_start":34,"highlight_end":38}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:163:34\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m163\u001b[0m \u001b[1m\u001b[94m|\u001b[0m doorbell_page as libc::off_t,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/queue.rs","byte_start":3575,"byte_end":3579,"line_start":96,"line_end":96,"column_start":26,"column_end":30,"is_primary":true,"text":[{"text":" _doorbell_mmap: *mut libc::c_void,","highlight_start":26,"highlight_end":30}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/queue.rs:96:26\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m96\u001b[0m \u001b[1m\u001b[94m|\u001b[0m _doorbell_mmap: *mut libc::c_void,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":11738,"byte_end":11742,"line_start":320,"line_end":320,"column_start":40,"column_end":44,"is_primary":true,"text":[{"text":" mmap_addr: cpu_ptr as *mut libc::c_void,","highlight_start":40,"highlight_end":44}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:320:40\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m320\u001b[0m \u001b[1m\u001b[94m|\u001b[0m mmap_addr: cpu_ptr as *mut libc::c_void,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"failed to resolve: use of unresolved module or unlinked crate `libc`","code":{"code":"E0433","explanation":"An undeclared crate, module, or type was used.\n\nErroneous code example:\n\n```compile_fail,E0433\nlet map = HashMap::new();\n// error: failed to resolve: use of undeclared type `HashMap`\n```\n\nPlease verify you didn't misspell the type/module's name or that you didn't\nforget to import it:\n\n```\nuse std::collections::HashMap; // HashMap has been imported.\nlet map: HashMap = HashMap::new(); // So it can be used!\n```\n\nIf you've expected to use a crate name:\n\n```compile_fail\nuse ferris_wheel::BigO;\n// error: failed to resolve: use of undeclared module or unlinked crate\n```\n\nMake sure the crate has been added as a dependency in `Cargo.toml`.\n\nTo use a module from your current crate, add the `crate::` prefix to the path.\n"},"level":"error","spans":[{"file_name":"src/memory.rs","byte_start":10996,"byte_end":11000,"line_start":298,"line_end":298,"column_start":37,"column_end":41,"is_primary":true,"text":[{"text":" mem.va_addr as *mut libc::c_void,","highlight_start":37,"highlight_end":41}],"label":"use of unresolved module or unlinked crate `libc`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[91merror[E0433]\u001b[0m\u001b[1m: failed to resolve: use of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/memory.rs:298:37\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m298\u001b[0m \u001b[1m\u001b[94m|\u001b[0m mem.va_addr as *mut libc::c_void,\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[91m^^^^\u001b[0m \u001b[1m\u001b[91muse of unresolved module or unlinked crate `libc`\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mhelp\u001b[0m: if you wanted to use a crate named `libc`, use `cargo add libc` to add it to your `Cargo.toml`\n\n"} +{"$message_type":"diagnostic","message":"aborting due to 47 previous errors; 1 warning emitted","code":null,"level":"error","spans":[],"children":[],"rendered":"\u001b[1m\u001b[91merror\u001b[0m\u001b[1m: aborting due to 47 previous errors; 1 warning emitted\u001b[0m\n\n"} +{"$message_type":"diagnostic","message":"For more information about this error, try `rustc --explain E0433`.","code":null,"level":"failure-note","spans":[],"children":[],"rendered":"\u001b[1mFor more information about this error, try `rustc --explain E0433`.\u001b[0m\n"} diff --git a/target/debug/.fingerprint/kfd-c036a4411ecb62c5/dep-lib-kfd b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/dep-lib-kfd new file mode 100644 index 0000000..1ed8d8a Binary files /dev/null and b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/dep-lib-kfd differ diff --git a/target/debug/.fingerprint/kfd-c036a4411ecb62c5/invoked.timestamp b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/invoked.timestamp new file mode 100644 index 0000000..e00328d --- /dev/null +++ b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/invoked.timestamp @@ -0,0 +1 @@ +This file has an mtime of when this was started. \ No newline at end of file diff --git a/target/debug/.fingerprint/kfd-c036a4411ecb62c5/lib-kfd b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/lib-kfd new file mode 100644 index 0000000..622ae5d --- /dev/null +++ b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/lib-kfd @@ -0,0 +1 @@ +da54f495bcf9c7e9 \ No newline at end of file diff --git a/target/debug/.fingerprint/kfd-c036a4411ecb62c5/lib-kfd.json b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/lib-kfd.json new file mode 100644 index 0000000..8aca7bf --- /dev/null +++ b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/lib-kfd.json @@ -0,0 +1 @@ +{"rustc":18276270781310494267,"features":"[]","declared_features":"[]","target":10069803095152535176,"profile":17672942494452627365,"path":10763286916239946207,"deps":[[12111499963430175700,"libc",false,1692664963012730200]],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/kfd-c036a4411ecb62c5/dep-lib-kfd","checksum":false}}],"rustflags":[],"config":8247474407144887393,"compile_kind":0} \ No newline at end of file diff --git a/target/debug/.fingerprint/kfd-c036a4411ecb62c5/output-lib-kfd b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/output-lib-kfd new file mode 100644 index 0000000..5195758 --- /dev/null +++ b/target/debug/.fingerprint/kfd-c036a4411ecb62c5/output-lib-kfd @@ -0,0 +1,4 @@ +{"$message_type":"diagnostic","message":"unused import: `KernelEntry`","code":{"code":"unused_imports","explanation":null},"level":"warning","spans":[{"file_name":"src/device.rs","byte_start":619,"byte_end":630,"line_start":17,"line_end":17,"column_start":57,"column_end":68,"is_primary":true,"text":[{"text":"use crate::dispatch::{CodeObject, GpuProgram, KernArgs, KernelEntry};","highlight_start":57,"highlight_end":68}],"label":null,"suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"`#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"remove the unused import","code":null,"level":"help","spans":[{"file_name":"src/device.rs","byte_start":617,"byte_end":630,"line_start":17,"line_end":17,"column_start":55,"column_end":68,"is_primary":true,"text":[{"text":"use crate::dispatch::{CodeObject, GpuProgram, KernArgs, KernelEntry};","highlight_start":55,"highlight_end":68}],"label":null,"suggested_replacement":"","suggestion_applicability":"MachineApplicable","expansion":null}],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[33mwarning\u001b[0m\u001b[1m: unused import: `KernelEntry`\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/device.rs:17:57\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m17\u001b[0m \u001b[1m\u001b[94m|\u001b[0m use crate::dispatch::{CodeObject, GpuProgram, KernArgs, KernelEntry};\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[33m^^^^^^^^^^^\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mnote\u001b[0m: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default\n\n"} +{"$message_type":"diagnostic","message":"constant `IOC_NONE` is never used","code":{"code":"dead_code","explanation":null},"level":"warning","spans":[{"file_name":"src/ioctl.rs","byte_start":321,"byte_end":329,"line_start":10,"line_end":10,"column_start":7,"column_end":15,"is_primary":true,"text":[{"text":"const IOC_NONE: u32 = 0;","highlight_start":7,"highlight_end":15}],"label":null,"suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"`#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default","code":null,"level":"note","spans":[],"children":[],"rendered":null}],"rendered":"\u001b[1m\u001b[33mwarning\u001b[0m\u001b[1m: constant `IOC_NONE` is never used\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/ioctl.rs:10:7\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m10\u001b[0m \u001b[1m\u001b[94m|\u001b[0m const IOC_NONE: u32 = 0;\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[33m^^^^^^^^\u001b[0m\n \u001b[1m\u001b[94m|\u001b[0m\n \u001b[1m\u001b[94m= \u001b[0m\u001b[1mnote\u001b[0m: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default\n\n"} +{"$message_type":"diagnostic","message":"constant `IOC_DESTROY_EVENT` is never used","code":{"code":"dead_code","explanation":null},"level":"warning","spans":[{"file_name":"src/ioctl.rs","byte_start":6326,"byte_end":6343,"line_start":221,"line_end":221,"column_start":7,"column_end":24,"is_primary":true,"text":[{"text":"const IOC_DESTROY_EVENT: u64 = iow::(0x09);","highlight_start":7,"highlight_end":24}],"label":null,"suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[],"rendered":"\u001b[1m\u001b[33mwarning\u001b[0m\u001b[1m: constant `IOC_DESTROY_EVENT` is never used\u001b[0m\n \u001b[1m\u001b[94m--> \u001b[0msrc/ioctl.rs:221:7\n \u001b[1m\u001b[94m|\u001b[0m\n\u001b[1m\u001b[94m221\u001b[0m \u001b[1m\u001b[94m|\u001b[0m const IOC_DESTROY_EVENT: u64 = iow::(0x09);\n \u001b[1m\u001b[94m|\u001b[0m \u001b[1m\u001b[33m^^^^^^^^^^^^^^^^^\u001b[0m\n\n"} +{"$message_type":"diagnostic","message":"3 warnings emitted","code":null,"level":"warning","spans":[],"children":[],"rendered":"\u001b[1m\u001b[33mwarning\u001b[0m\u001b[1m: 3 warnings emitted\u001b[0m\n\n"} diff --git a/target/debug/.fingerprint/libc-184a2b4f65324410/dep-lib-libc b/target/debug/.fingerprint/libc-184a2b4f65324410/dep-lib-libc new file mode 100644 index 0000000..ec3cb8b Binary files /dev/null and b/target/debug/.fingerprint/libc-184a2b4f65324410/dep-lib-libc differ diff --git a/target/debug/.fingerprint/libc-184a2b4f65324410/invoked.timestamp b/target/debug/.fingerprint/libc-184a2b4f65324410/invoked.timestamp new file mode 100644 index 0000000..e00328d --- /dev/null +++ b/target/debug/.fingerprint/libc-184a2b4f65324410/invoked.timestamp @@ -0,0 +1 @@ +This file has an mtime of when this was started. \ No newline at end of file diff --git a/target/debug/.fingerprint/libc-184a2b4f65324410/lib-libc b/target/debug/.fingerprint/libc-184a2b4f65324410/lib-libc new file mode 100644 index 0000000..926ca8d --- /dev/null +++ b/target/debug/.fingerprint/libc-184a2b4f65324410/lib-libc @@ -0,0 +1 @@ +589d1fd4d08d7d17 \ No newline at end of file diff --git a/target/debug/.fingerprint/libc-184a2b4f65324410/lib-libc.json b/target/debug/.fingerprint/libc-184a2b4f65324410/lib-libc.json new file mode 100644 index 0000000..2bf6626 --- /dev/null +++ b/target/debug/.fingerprint/libc-184a2b4f65324410/lib-libc.json @@ -0,0 +1 @@ +{"rustc":18276270781310494267,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":17682796336736096309,"profile":15222631470922254920,"path":3187973484572868484,"deps":[[12111499963430175700,"build_script_build",false,15831001214597677411]],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/libc-184a2b4f65324410/dep-lib-libc","checksum":false}}],"rustflags":[],"config":8247474407144887393,"compile_kind":0} \ No newline at end of file diff --git a/target/debug/.fingerprint/libc-6578899ebd06082d/build-script-build-script-build b/target/debug/.fingerprint/libc-6578899ebd06082d/build-script-build-script-build new file mode 100644 index 0000000..34c64cf --- /dev/null +++ b/target/debug/.fingerprint/libc-6578899ebd06082d/build-script-build-script-build @@ -0,0 +1 @@ +e4b79e7c27b422e1 \ No newline at end of file diff --git a/target/debug/.fingerprint/libc-6578899ebd06082d/build-script-build-script-build.json b/target/debug/.fingerprint/libc-6578899ebd06082d/build-script-build-script-build.json new file mode 100644 index 0000000..5341b23 --- /dev/null +++ b/target/debug/.fingerprint/libc-6578899ebd06082d/build-script-build-script-build.json @@ -0,0 +1 @@ +{"rustc":18276270781310494267,"features":"[\"default\", \"std\"]","declared_features":"[\"align\", \"const-extern-fn\", \"default\", \"extra_traits\", \"rustc-dep-of-std\", \"rustc-std-workspace-core\", \"std\", \"use_std\"]","target":5408242616063297496,"profile":1565149285177326037,"path":1233441601326685821,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/libc-6578899ebd06082d/dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":8247474407144887393,"compile_kind":0} \ No newline at end of file diff --git a/target/debug/.fingerprint/libc-6578899ebd06082d/dep-build-script-build-script-build b/target/debug/.fingerprint/libc-6578899ebd06082d/dep-build-script-build-script-build new file mode 100644 index 0000000..ec3cb8b Binary files /dev/null and b/target/debug/.fingerprint/libc-6578899ebd06082d/dep-build-script-build-script-build differ diff --git a/target/debug/.fingerprint/libc-6578899ebd06082d/invoked.timestamp b/target/debug/.fingerprint/libc-6578899ebd06082d/invoked.timestamp new file mode 100644 index 0000000..e00328d --- /dev/null +++ b/target/debug/.fingerprint/libc-6578899ebd06082d/invoked.timestamp @@ -0,0 +1 @@ +This file has an mtime of when this was started. \ No newline at end of file diff --git a/target/debug/.fingerprint/libc-af64a192d6d29af3/run-build-script-build-script-build b/target/debug/.fingerprint/libc-af64a192d6d29af3/run-build-script-build-script-build new file mode 100644 index 0000000..99a5064 --- /dev/null +++ b/target/debug/.fingerprint/libc-af64a192d6d29af3/run-build-script-build-script-build @@ -0,0 +1 @@ +6341df7aba03b3db \ No newline at end of file diff --git a/target/debug/.fingerprint/libc-af64a192d6d29af3/run-build-script-build-script-build.json b/target/debug/.fingerprint/libc-af64a192d6d29af3/run-build-script-build-script-build.json new file mode 100644 index 0000000..2495c25 --- /dev/null +++ b/target/debug/.fingerprint/libc-af64a192d6d29af3/run-build-script-build-script-build.json @@ -0,0 +1 @@ +{"rustc":18276270781310494267,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[12111499963430175700,"build_script_build",false,16222726889429448676]],"local":[{"RerunIfChanged":{"output":"debug/build/libc-af64a192d6d29af3/output","paths":["build.rs"]}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_FREEBSD_VERSION","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_MUSL_V1_2_3","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_LINUX_TIME_BITS64","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_GNU_FILE_OFFSET_BITS","val":null}},{"RerunIfEnvChanged":{"var":"RUST_LIBC_UNSTABLE_GNU_TIME_BITS","val":null}}],"rustflags":[],"config":0,"compile_kind":0} \ No newline at end of file diff --git a/target/debug/build/libc-6578899ebd06082d/build-script-build b/target/debug/build/libc-6578899ebd06082d/build-script-build new file mode 100755 index 0000000..4643748 Binary files /dev/null and b/target/debug/build/libc-6578899ebd06082d/build-script-build differ diff --git a/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d b/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d new file mode 100755 index 0000000..4643748 Binary files /dev/null and b/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d differ diff --git a/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d.d b/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d.d new file mode 100644 index 0000000..485e36a --- /dev/null +++ b/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d.d @@ -0,0 +1,5 @@ +/steam/rotko/kfd/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d.d: /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/build.rs + +/steam/rotko/kfd/target/debug/build/libc-6578899ebd06082d/build_script_build-6578899ebd06082d: /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/build.rs + +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/build.rs: diff --git a/target/debug/build/libc-af64a192d6d29af3/invoked.timestamp b/target/debug/build/libc-af64a192d6d29af3/invoked.timestamp new file mode 100644 index 0000000..e00328d --- /dev/null +++ b/target/debug/build/libc-af64a192d6d29af3/invoked.timestamp @@ -0,0 +1 @@ +This file has an mtime of when this was started. \ No newline at end of file diff --git a/target/debug/build/libc-af64a192d6d29af3/output b/target/debug/build/libc-af64a192d6d29af3/output new file mode 100644 index 0000000..89a43b5 --- /dev/null +++ b/target/debug/build/libc-af64a192d6d29af3/output @@ -0,0 +1,25 @@ +cargo:rerun-if-changed=build.rs +cargo:rerun-if-env-changed=RUST_LIBC_UNSTABLE_FREEBSD_VERSION +cargo:rustc-cfg=freebsd12 +cargo:rerun-if-env-changed=RUST_LIBC_UNSTABLE_MUSL_V1_2_3 +cargo:rerun-if-env-changed=RUST_LIBC_UNSTABLE_LINUX_TIME_BITS64 +cargo:rerun-if-env-changed=RUST_LIBC_UNSTABLE_GNU_FILE_OFFSET_BITS +cargo:rerun-if-env-changed=RUST_LIBC_UNSTABLE_GNU_TIME_BITS +cargo:rustc-check-cfg=cfg(emscripten_old_stat_abi) +cargo:rustc-check-cfg=cfg(espidf_time32) +cargo:rustc-check-cfg=cfg(freebsd10) +cargo:rustc-check-cfg=cfg(freebsd11) +cargo:rustc-check-cfg=cfg(freebsd12) +cargo:rustc-check-cfg=cfg(freebsd13) +cargo:rustc-check-cfg=cfg(freebsd14) +cargo:rustc-check-cfg=cfg(freebsd15) +cargo:rustc-check-cfg=cfg(gnu_file_offset_bits64) +cargo:rustc-check-cfg=cfg(gnu_time_bits64) +cargo:rustc-check-cfg=cfg(libc_deny_warnings) +cargo:rustc-check-cfg=cfg(linux_time_bits64) +cargo:rustc-check-cfg=cfg(musl_v1_2_3) +cargo:rustc-check-cfg=cfg(musl32_time64) +cargo:rustc-check-cfg=cfg(vxworks_lt_25_09) +cargo:rustc-check-cfg=cfg(target_os,values("switch","aix","ohos","hurd","rtems","visionos","nuttx","cygwin","qurt")) +cargo:rustc-check-cfg=cfg(target_env,values("illumos","wasi","aix","ohos","nto71_iosock","nto80")) +cargo:rustc-check-cfg=cfg(target_arch,values("loongarch64","mips32r6","mips64r6","csky")) diff --git a/target/debug/build/libc-af64a192d6d29af3/root-output b/target/debug/build/libc-af64a192d6d29af3/root-output new file mode 100644 index 0000000..155b4cd --- /dev/null +++ b/target/debug/build/libc-af64a192d6d29af3/root-output @@ -0,0 +1 @@ +/steam/rotko/kfd/target/debug/build/libc-af64a192d6d29af3/out \ No newline at end of file diff --git a/target/debug/build/libc-af64a192d6d29af3/stderr b/target/debug/build/libc-af64a192d6d29af3/stderr new file mode 100644 index 0000000..e69de29 diff --git a/target/debug/deps/kfd-2447e9870f0e8752.d b/target/debug/deps/kfd-2447e9870f0e8752.d new file mode 100644 index 0000000..cb4f1a6 --- /dev/null +++ b/target/debug/deps/kfd-2447e9870f0e8752.d @@ -0,0 +1,16 @@ +/steam/rotko/kfd/target/debug/deps/kfd-2447e9870f0e8752.d: src/lib.rs src/ioctl.rs src/memory.rs src/queue.rs src/dispatch.rs src/compute.rs src/device.rs src/kernels/test_store.co src/kernels/matvec_asm.co src/kernels/matmul_blocked.co src/kernels/matmul_small.co src/kernels/superlinear.co + +/steam/rotko/kfd/target/debug/deps/libkfd-2447e9870f0e8752.rmeta: src/lib.rs src/ioctl.rs src/memory.rs src/queue.rs src/dispatch.rs src/compute.rs src/device.rs src/kernels/test_store.co src/kernels/matvec_asm.co src/kernels/matmul_blocked.co src/kernels/matmul_small.co src/kernels/superlinear.co + +src/lib.rs: +src/ioctl.rs: +src/memory.rs: +src/queue.rs: +src/dispatch.rs: +src/compute.rs: +src/device.rs: +src/kernels/test_store.co: +src/kernels/matvec_asm.co: +src/kernels/matmul_blocked.co: +src/kernels/matmul_small.co: +src/kernels/superlinear.co: diff --git a/target/debug/deps/kfd-c036a4411ecb62c5.d b/target/debug/deps/kfd-c036a4411ecb62c5.d new file mode 100644 index 0000000..a064797 --- /dev/null +++ b/target/debug/deps/kfd-c036a4411ecb62c5.d @@ -0,0 +1,16 @@ +/steam/rotko/kfd/target/debug/deps/kfd-c036a4411ecb62c5.d: src/lib.rs src/ioctl.rs src/memory.rs src/queue.rs src/dispatch.rs src/compute.rs src/device.rs src/kernels/test_store.co src/kernels/matvec_asm.co src/kernels/matmul_blocked.co src/kernels/matmul_small.co src/kernels/superlinear.co + +/steam/rotko/kfd/target/debug/deps/libkfd-c036a4411ecb62c5.rmeta: src/lib.rs src/ioctl.rs src/memory.rs src/queue.rs src/dispatch.rs src/compute.rs src/device.rs src/kernels/test_store.co src/kernels/matvec_asm.co src/kernels/matmul_blocked.co src/kernels/matmul_small.co src/kernels/superlinear.co + +src/lib.rs: +src/ioctl.rs: +src/memory.rs: +src/queue.rs: +src/dispatch.rs: +src/compute.rs: +src/device.rs: +src/kernels/test_store.co: +src/kernels/matvec_asm.co: +src/kernels/matmul_blocked.co: +src/kernels/matmul_small.co: +src/kernels/superlinear.co: diff --git a/target/debug/deps/libc-184a2b4f65324410.d b/target/debug/deps/libc-184a2b4f65324410.d new file mode 100644 index 0000000..c313e8c --- /dev/null +++ b/target/debug/deps/libc-184a2b4f65324410.d @@ -0,0 +1,44 @@ +/steam/rotko/kfd/target/debug/deps/libc-184a2b4f65324410.d: /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/lib.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/macros.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/linux_like/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/linux_like/pthread.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/pthread.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/unistd.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/bcm.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/error.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/j1939.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/netlink.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/raw.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/keyctl.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/membarrier.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/netlink.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/pidfd.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/posix/unistd.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/nptl/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/nptl/pthread.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/linux/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/linux/net/route.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/primitives.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/arch/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux_l4re_shared.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/x86_64/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/x86_64/not_x32.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/arch/generic/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/types.rs + +/steam/rotko/kfd/target/debug/deps/liblibc-184a2b4f65324410.rmeta: /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/lib.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/macros.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/linux_like/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/linux_like/pthread.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/pthread.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/unistd.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/bcm.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/error.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/j1939.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/netlink.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/raw.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/keyctl.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/membarrier.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/netlink.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/pidfd.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/posix/unistd.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/nptl/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/nptl/pthread.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/linux/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/linux/net/route.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/primitives.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/arch/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux_l4re_shared.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/x86_64/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/x86_64/not_x32.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/arch/generic/mod.rs /home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/types.rs + +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/lib.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/macros.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/linux_like/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/linux_like/pthread.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/pthread.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/common/posix/unistd.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/bcm.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/error.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/j1939.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/netlink.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/can/raw.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/keyctl.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/membarrier.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/netlink.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/linux_uapi/linux/pidfd.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/posix/unistd.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/nptl/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/nptl/pthread.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/linux/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/new/glibc/sysdeps/unix/linux/net/route.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/primitives.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/arch/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux_l4re_shared.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/x86_64/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/gnu/b64/x86_64/not_x32.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/unix/linux_like/linux/arch/generic/mod.rs: +/home/alice/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.184/src/types.rs: diff --git a/target/debug/deps/libkfd-c036a4411ecb62c5.rmeta b/target/debug/deps/libkfd-c036a4411ecb62c5.rmeta new file mode 100644 index 0000000..050dab8 Binary files /dev/null and b/target/debug/deps/libkfd-c036a4411ecb62c5.rmeta differ diff --git a/target/debug/deps/liblibc-184a2b4f65324410.rmeta b/target/debug/deps/liblibc-184a2b4f65324410.rmeta new file mode 100644 index 0000000..04282ce Binary files /dev/null and b/target/debug/deps/liblibc-184a2b4f65324410.rmeta differ diff --git a/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/dep-graph.bin b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/dep-graph.bin new file mode 100644 index 0000000..798cdcf Binary files /dev/null and b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/dep-graph.bin differ diff --git a/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/metadata.rmeta b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/metadata.rmeta new file mode 100644 index 0000000..050dab8 Binary files /dev/null and b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/metadata.rmeta differ diff --git a/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/query-cache.bin b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/query-cache.bin new file mode 100644 index 0000000..060e9e1 Binary files /dev/null and b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/query-cache.bin differ diff --git a/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/work-products.bin b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/work-products.bin new file mode 100644 index 0000000..cbee428 Binary files /dev/null and b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb-0nanvm67qve0kwcedetfc5y1i/work-products.bin differ diff --git a/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb.lock b/target/debug/incremental/kfd-2a7rsuw39z8ru/s-hhcthcl33v-0whmljb.lock new file mode 100644 index 0000000..e69de29 diff --git a/target/debug/incremental/kfd-3vh5csval5g5u/s-hhcth4cuny-11e5e6s-working/dep-graph.part.bin b/target/debug/incremental/kfd-3vh5csval5g5u/s-hhcth4cuny-11e5e6s-working/dep-graph.part.bin new file mode 100644 index 0000000..2a20462 Binary files /dev/null and b/target/debug/incremental/kfd-3vh5csval5g5u/s-hhcth4cuny-11e5e6s-working/dep-graph.part.bin differ diff --git a/target/debug/incremental/kfd-3vh5csval5g5u/s-hhcth4cuny-11e5e6s.lock b/target/debug/incremental/kfd-3vh5csval5g5u/s-hhcth4cuny-11e5e6s.lock new file mode 100644 index 0000000..e69de29 diff --git a/tests/kfd_bench.rs b/tests/kfd_bench.rs new file mode 100644 index 0000000..64721cd --- /dev/null +++ b/tests/kfd_bench.rs @@ -0,0 +1,160 @@ +//! kfd gpu performance benchmarks. +//! run: cargo test --release --test kfd_bench -- --nocapture --test-threads=1 +//! +//! measures gpu dispatch latency vs cpu at various matrix sizes +//! to find the crossover point where gpu wins. + +use isis::host::device::kfd::{self, HsaDevice}; +use isis::host::device::kfd::dispatch::KernArgs; +use isis::organism::compute::{ComputeBackend, CpuBackend}; +use std::time::Instant; + +fn open_gpu() -> Option> { + use std::sync::{OnceLock, Mutex}; + static GPU: OnceLock>> = OnceLock::new(); + GPU.get_or_init(|| { + if !kfd::is_available() { return None; } + HsaDevice::open().ok().map(Mutex::new) + }).as_ref().map(|m| m.lock().unwrap()) +} + +/// cpu reference matvec +fn cpu_matvec_timed(cpu: &CpuBackend, w: &[f32], b: &[f32], x: &[f32], + y: &mut [f32], out: usize, inp: usize, iters: usize) -> f64 { + let t = Instant::now(); + for _ in 0..iters { + cpu.matvec(w, b, x, y, out, inp); + } + t.elapsed().as_micros() as f64 / iters as f64 +} + +/// gpu matvec — allocate once, dispatch N times +fn gpu_matvec_timed(dev: &mut HsaDevice, w: &[f32], b: &[f32], x: &[f32], + out: u32, inp: u32, iters: usize) -> f64 { + let w_buf = dev.upload_f32(w).unwrap(); + let b_buf = dev.upload_f32(b).unwrap(); + let x_buf = dev.upload_f32(x).unwrap(); + let y_buf = dev.alloc.alloc_userptr_public(((out as u64) * 4 + 4095) & !4095).unwrap(); + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); + args.push_ptr(&b_buf); + args.push_ptr(&x_buf); + args.push_ptr(&y_buf); + args.push_u32(out); + args.push_u32(inp); + let args_buf = args.upload(&dev.alloc).unwrap(); + let block = out.min(256); + + // warmup + dev.dispatch_kernel("matvec", &args_buf, [out, 1, 1], [block, 1, 1]); + + let t = Instant::now(); + for _ in 0..iters { + dev.dispatch_kernel("matvec", &args_buf, [out, 1, 1], [block, 1, 1]); + } + t.elapsed().as_micros() as f64 / iters as f64 +} + +#[test] +fn bench_matvec_sizes() { + let mut dev = match open_gpu() { Some(d) => d, None => { + eprintln!(" SKIP: no gpu"); return; + }}; + if dev.kernels.is_none() { eprintln!(" SKIP: no kernels"); return; } + let cpu = CpuBackend::new(); + + println!(); + println!(" {:>6} x {:<6} {:>10} {:>10} {:>8} winner", "out", "in", "cpu (us)", "gpu (us)", "speedup"); + println!(" {}", "-".repeat(62)); + + for &(out, inp) in &[ + (4, 4), (8, 8), (16, 16), (32, 32), (64, 64), + (128, 128), (256, 256), (512, 512), + (1024, 1024), + (64, 1024), (1024, 64), + ] { + let n = out * inp; + let w: Vec = (0..n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect(); + let b: Vec = (0..out).map(|i| (i % 10) as f32 * 0.1).collect(); + let x: Vec = (0..inp).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect(); + let mut y_cpu = vec![0.0f32; out]; + + let iters = if n < 10000 { 1000 } else if n < 100000 { 100 } else { 10 }; + + let cpu_us = cpu_matvec_timed(&cpu, &w, &b, &x, &mut y_cpu, out, inp, iters); + let gpu_us = gpu_matvec_timed(&mut dev, &w, &b, &x, out as u32, inp as u32, iters); + + let speedup = cpu_us / gpu_us; + let winner = if gpu_us < cpu_us { "gpu" } else { "cpu" }; + + println!(" {:>6} x {:<6} {:>10.1} {:>10.1} {:>7.2}x {}", + out, inp, cpu_us, gpu_us, speedup, winner); + } + println!(); + + // isis-relevant sizes (from CTM config) + println!(" isis workload sizes:"); + println!(" {:>6} x {:<6} {:>10} {:>10} {:>8} context", "out", "in", "cpu (us)", "gpu (us)", "speedup"); + println!(" {}", "-".repeat(72)); + + let isis_sizes = [ + (128, 20, "input synapse (d_input=20 → 128 neurons)"), + (256, 128, "attention synapse (128 → 256)"), + (256, 256, "output synapse (256 → 256)"), + (128, 256, "motor synapse (256 → 128)"), + (1024, 20, "large input (d_input=20 → 1024)"), + (1024, 1024, "large attention (1024 → 1024)"), + (4096, 1024, "xl attention (1024 → 4096)"), + (256, 4096, "sync accumulator readout"), + ]; + + for &(out, inp, label) in &isis_sizes { + let n = out * inp; + let w: Vec = (0..n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect(); + let b: Vec = (0..out).map(|i| (i % 10) as f32 * 0.1).collect(); + let x: Vec = (0..inp).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect(); + let mut y_cpu = vec![0.0f32; out]; + + let iters = if n < 10000 { 1000 } else if n < 100000 { 100 } else { 10 }; + + let cpu_us = cpu_matvec_timed(&cpu, &w, &b, &x, &mut y_cpu, out, inp, iters); + let gpu_us = gpu_matvec_timed(&mut dev, &w, &b, &x, out as u32, inp as u32, iters); + + let speedup = cpu_us / gpu_us; + println!(" {:>6} x {:<6} {:>10.1} {:>10.1} {:>7.2}x {}", + out, inp, cpu_us, gpu_us, speedup, label); + } + // llm-scale matmul sizes (matvec = batch=1 inference) + println!(" llm inference sizes (batch=1 matvec):"); + println!(" {:>6} x {:<6} {:>10} {:>10} {:>8} context", "out", "in", "cpu (us)", "gpu (us)", "speedup"); + println!(" {}", "-".repeat(72)); + + let llm_sizes = [ + (896, 896, "qwen2.5-0.5B hidden"), + (4864, 896, "qwen2.5-0.5B mlp up"), + (896, 4864, "qwen2.5-0.5B mlp down"), + (2048, 2048, "llama-1B hidden"), + (5632, 2048, "llama-1B mlp up"), + ]; + + for &(out, inp, label) in &llm_sizes { + let n = out * inp; + let w: Vec = (0..n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect(); + let b = vec![0.0f32; out]; + let x: Vec = (0..inp).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect(); + let mut y_cpu = vec![0.0f32; out]; + + let iters = 10; + let cpu_us = cpu_matvec_timed(&cpu, &w, &b, &x, &mut y_cpu, out, inp, iters); + let gpu_us = gpu_matvec_timed(&mut dev, &w, &b, &x, out as u32, inp as u32, iters); + + let speedup = cpu_us / gpu_us; + let flops = (out * inp * 2) as f64; + let gpu_gflops = flops / (gpu_us * 1e3); // GFLOP/s + let cpu_gflops = flops / (cpu_us * 1e3); + println!(" {:>6} x {:<6} {:>8.0} ({:>4.1} GF/s) {:>8.0} ({:>4.1} GF/s) {:>5.2}x {}", + out, inp, cpu_us, cpu_gflops, gpu_us, gpu_gflops, speedup, label); + } + println!(); +} diff --git a/tests/kfd_gpu.rs b/tests/kfd_gpu.rs new file mode 100644 index 0000000..b1201f7 --- /dev/null +++ b/tests/kfd_gpu.rs @@ -0,0 +1,198 @@ +//! kfd gpu driver integration tests. +//! +//! run: cargo test --release --test kfd_gpu -- --nocapture +//! +//! these tests require /dev/kfd (amd gpu with kfd driver). +//! they share a single HsaDevice to avoid acquire_vm conflicts. +//! each test validates correctness against a cpu reference. + +use isis::host::device::kfd::{self, HsaDevice}; +use isis::host::device::kfd::dispatch::KernArgs; +use std::sync::OnceLock; + +/// single shared gpu device for all tests. +/// tests run sequentially (--test-threads=1 enforced by kfd constraint). +static GPU: OnceLock>> = OnceLock::new(); + +fn gpu() -> Option> { + GPU.get_or_init(|| { + if !kfd::is_available() { return None; } + HsaDevice::open().ok().map(std::sync::Mutex::new) + }).as_ref().map(|m| m.lock().unwrap()) +} + +// ─── store kernel ─────────────────────────────────────────── + +#[test] +fn store42_writes_correct_values() { + let mut dev = match gpu() { Some(d) => d, None => return }; + + let n = 32u32; + let y = dev.alloc.alloc_userptr_public((n as u64) * 4).unwrap(); + // write sentinel — must be overwritten + y.write_f32(0, &vec![-1.0f32; n as usize]); + + let mut args = KernArgs::new(); + args.push_ptr(&y); + let args_buf = args.upload(&dev.alloc).unwrap(); + + assert!(dev.dispatch_kernel("test_store", &args_buf, [n, 1, 1], [n, 1, 1]), + "dispatch timed out"); + + let result = y.read_f32(0, n as usize); + for i in 0..n as usize { + assert_eq!(result[i], 42.0, "y[{i}] = {} (sentinel was -1.0)", result[i]); + } +} + +// ─── matvec kernel ────────────────────────────────────────── + +/// dispatch matvec and return output vector. +fn gpu_matvec(dev: &mut HsaDevice, w: &[f32], b: &[f32], x: &[f32], + out_dim: u32, in_dim: u32) -> Vec { + let w_buf = dev.upload_f32(w).unwrap(); + let b_buf = dev.upload_f32(b).unwrap(); + let x_buf = dev.upload_f32(x).unwrap(); + let y_buf = dev.alloc.alloc_userptr_public(((out_dim as u64) * 4 + 4095) & !4095).unwrap(); + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); + args.push_ptr(&b_buf); + args.push_ptr(&x_buf); + args.push_ptr(&y_buf); + args.push_u32(out_dim); + args.push_u32(in_dim); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let block = out_dim.min(256); + assert!(dev.dispatch_kernel("matvec", &args_buf, [out_dim, 1, 1], [block, 1, 1]), + "matvec dispatch timed out ({}x{})", out_dim, in_dim); + + y_buf.read_f32(0, out_dim as usize) +} + +/// cpu reference: y = W*x + b +fn cpu_matvec(w: &[f32], b: &[f32], x: &[f32], out_dim: usize, in_dim: usize) -> Vec { + let mut y = vec![0.0f32; out_dim]; + for row in 0..out_dim { + let mut sum = b[row]; + for col in 0..in_dim { + sum += w[row * in_dim + col] * x[col]; + } + y[row] = sum; + } + y +} + +fn assert_close(gpu: &[f32], cpu: &[f32], tol: f32, label: &str) { + assert_eq!(gpu.len(), cpu.len(), "{label}: length mismatch {} vs {}", gpu.len(), cpu.len()); + for i in 0..gpu.len() { + let err = (gpu[i] - cpu[i]).abs(); + assert!(err < tol, "{label}: y[{i}] gpu={} cpu={} err={err}", gpu[i], cpu[i]); + } +} + +#[test] +fn matvec_identity_4x4() { + let mut dev = match gpu() { Some(d) => d, None => return }; + let n = 4; + let mut w = vec![0.0f32; n * n]; + for i in 0..n { w[i * n + i] = 1.0; } + let b = vec![0.5; n]; + let x = vec![1.0, 2.0, 3.0, 4.0]; + + let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, n as u32, n as u32); + let cpu_y = cpu_matvec(&w, &b, &x, n, n); + assert_close(&gpu_y, &cpu_y, 1e-5, "identity_4x4"); +} + +#[test] +fn matvec_dense_2x3() { + let mut dev = match gpu() { Some(d) => d, None => return }; + let w = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0f32]; + let b = vec![0.1, 0.2]; + let x = vec![1.0, 1.0, 1.0]; + + let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, 2, 3); + let cpu_y = cpu_matvec(&w, &b, &x, 2, 3); + assert_close(&gpu_y, &cpu_y, 1e-4, "dense_2x3"); +} + +#[test] +fn matvec_32x32_vs_cpu() { + let mut dev = match gpu() { Some(d) => d, None => return }; + let n = 32usize; + + // deterministic pseudo-random weights + let w: Vec = (0..n*n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect(); + let b: Vec = (0..n).map(|i| (i % 10) as f32 * 0.1).collect(); + let x: Vec = (0..n).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect(); + + let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, n as u32, n as u32); + let cpu_y = cpu_matvec(&w, &b, &x, n, n); + assert_close(&gpu_y, &cpu_y, 1e-3, "random_32x32"); +} + +#[test] +fn matvec_nonsquare_8x16() { + let mut dev = match gpu() { Some(d) => d, None => return }; + let out = 8usize; + let inp = 16usize; + + let w = vec![1.0f32; out * inp]; // all ones + let b = vec![0.0; out]; + let x = vec![1.0; inp]; // all ones + + let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, out as u32, inp as u32); + let cpu_y = cpu_matvec(&w, &b, &x, out, inp); + assert_close(&gpu_y, &cpu_y, 1e-3, "nonsquare_8x16"); + + // each row should sum to 16.0 (16 ones) + for i in 0..out { + assert!((gpu_y[i] - 16.0).abs() < 1e-3, "y[{i}] = {}", gpu_y[i]); + } +} + +#[test] +fn matvec_scalar_1x1() { + let mut dev = match gpu() { Some(d) => d, None => return }; + let gpu_y = gpu_matvec(&mut dev, &[3.0], &[0.5], &[2.0], 1, 1); + assert!((gpu_y[0] - 6.5).abs() < 1e-5, "1x1: {} expected 6.5", gpu_y[0]); +} + +// ─── async future ─────────────────────────────────────────── + +#[test] +fn dispatch_async_returns_future() { + use isis::host::device::kfd::compute::GpuFuture; + + let mut dev = match gpu() { Some(d) => d, None => return }; + + let y = dev.alloc.alloc_userptr_public(4096).unwrap(); + let mut args = KernArgs::new(); + args.push_ptr(&y); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let future = dev.dispatch_async("test_store", &args_buf, [4, 1, 1], [4, 1, 1]); + assert!(future.is_some(), "dispatch_async returned None"); + + let future = future.unwrap(); + assert!(!future.poll() || true, "poll should not panic"); // may already be done + let elapsed = future.wait(1_000_000); + assert!(elapsed.is_some(), "future timed out after 1s"); + + let result = y.read_f32(0, 4); + assert_eq!(result, vec![42.0; 4]); +} + +#[test] +fn dispatch_nonexistent_kernel_returns_none() { + let mut dev = match gpu() { Some(d) => d, None => return }; + let y = dev.alloc.alloc_userptr_public(4096).unwrap(); + let mut args = KernArgs::new(); + args.push_ptr(&y); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let future = dev.dispatch_async("nonexistent_kernel", &args_buf, [1, 1, 1], [1, 1, 1]); + assert!(future.is_none(), "should return None for unknown kernel"); +} diff --git a/tests/kfd_matmul_bench.rs b/tests/kfd_matmul_bench.rs new file mode 100644 index 0000000..6d16851 --- /dev/null +++ b/tests/kfd_matmul_bench.rs @@ -0,0 +1,167 @@ +//! matmul kernel benchmark — pure rust dispatch. +//! run: cargo test --release --test kfd_matmul_bench -- --nocapture --test-threads=1 + +use isis::host::device::kfd::{self, HsaDevice}; +use isis::host::device::kfd::dispatch::{CodeObject, KernArgs}; +use isis::host::device::kfd::memory::GpuBuffer; +use std::time::Instant; + +static MATMUL_BLOCKED_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co"); +static MATMUL_SMALL_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_small.co"); + +#[test] +fn matmul_bench() { + if !kfd::is_available() { eprintln!("skip: no kfd"); return; } + let mut dev = match HsaDevice::open() { Ok(d) => d, Err(e) => { eprintln!("skip: {e}"); return; }}; + + // Load both matmul kernels + for co_bytes in [MATMUL_BLOCKED_CO, MATMUL_SMALL_CO] { + let co = CodeObject::load(&dev.alloc, co_bytes).unwrap(); + if dev.kernels.is_none() { dev.kernels = Some(std::collections::HashMap::new()); } + for (name, entry) in &co.kernels { + dev.kernels.as_mut().unwrap().insert(name.clone(), entry.clone()); + } + std::mem::forget(co); // keep code in VRAM + } + + if let Some(mhz) = dev.current_sclk_mhz() { + println!(" sclk: {} mhz", mhz); + } + println!(); + + // ---- Correctness check for matmul_blocked (TM=128) ---- + { + let m = 128u32; let k = 8u32; let n = 32u32; + let mut w_data = vec![0.0f32; (m * k) as usize]; + for i in 0..m as usize { w_data[i * k as usize] = (i + 1) as f32; } + let b_data = vec![0.0f32; m as usize]; + let mut x_data = vec![0.0f32; (n * k) as usize]; + for j in 0..n as usize { x_data[j * k as usize] = 1.0; } + + let w_buf = dev.upload_f32(&w_data).unwrap(); + let b_buf = dev.upload_f32(&b_data).unwrap(); + let x_buf = dev.upload_f32(&x_data).unwrap(); + let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap(); + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); args.push_ptr(&b_buf); + args.push_ptr(&x_buf); args.push_ptr(&y_buf); + args.push_u32(m); args.push_u32(k); args.push_u32(n); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let nwg = ((m + 127) / 128) * ((n + 31) / 32); + dev.dispatch_kernel("matmul_blocked", &args_buf, [nwg, 1, 1], [256, 1, 1]); + + let y_slice = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr as *const f32, (n * m) as usize) }; + print!(" matmul_blocked (TM=128): "); + let mut ok = true; + for j in 0..n as usize { + for i in 0..m as usize { + let expected = (i + 1) as f32; + if (y_slice[j * m as usize + i] - expected).abs() > 0.1 { ok = false; break; } + } + if !ok { break; } + } + println!("{}", if ok { "PASS" } else { "FAIL" }); + assert!(ok, "matmul_blocked correctness failed"); + } + + // ---- Correctness check for matmul_small (TM=32) ---- + { + let m = 32u32; let k = 8u32; let n = 32u32; + let mut w_data = vec![0.0f32; (m * k) as usize]; + for i in 0..m as usize { w_data[i * k as usize] = (i + 1) as f32; } + let b_data = vec![0.0f32; m as usize]; + let mut x_data = vec![0.0f32; (n * k) as usize]; + for j in 0..n as usize { x_data[j * k as usize] = 1.0; } + + let w_buf = dev.upload_f32_col_major(&w_data, m as usize, k as usize).unwrap(); + let b_buf = dev.upload_f32(&b_data).unwrap(); + let x_buf = dev.upload_f32(&x_data).unwrap(); + let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap(); + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); args.push_ptr(&b_buf); + args.push_ptr(&x_buf); args.push_ptr(&y_buf); + args.push_u32(m); args.push_u32(k); args.push_u32(n); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let nwg = ((m + 31) / 32) * ((n + 31) / 32); + dev.dispatch_kernel("matmul_small", &args_buf, [nwg, 1, 1], [256, 1, 1]); + + let y_slice = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr as *const f32, (n * m) as usize) }; + print!(" matmul_small (TM=32): "); + let mut ok = true; + for j in 0..n as usize { + for i in 0..m as usize { + let expected = (i + 1) as f32; + if (y_slice[j * m as usize + i] - expected).abs() > 0.1 { ok = false; break; } + } + if !ok { break; } + } + println!("{}", if ok { "PASS" } else { "FAIL" }); + assert!(ok, "matmul_small correctness failed"); + } + println!(); + + // ---- Performance benchmark: both kernels + dispatch selector ---- + let shapes: &[(u32, u32, u32, &str)] = &[ + (512, 512, 32, ""), + (1024, 1024, 32, ""), + (2048, 2048, 32, ""), + (4096, 4096, 32, "qwen attn"), + (4096, 11008, 32, "qwen mlp"), + (4864, 896, 32, "isis layer"), + ]; + + // bench helper + let bench_kernel = |dev: &mut HsaDevice, kernel: &str, m: u32, k: u32, n: u32, + w_buf: &GpuBuffer, b_buf: &GpuBuffer, x_buf: &GpuBuffer, y_buf: &GpuBuffer| -> f64 { + let mut args = KernArgs::new(); + args.push_ptr(w_buf); args.push_ptr(b_buf); + args.push_ptr(x_buf); args.push_ptr(y_buf); + args.push_u32(m); args.push_u32(k); args.push_u32(n); + let args_buf = args.upload(&dev.alloc).unwrap(); + + let (nwg, block) = if kernel == "matmul_blocked" { + (((m + 127) / 128) * ((n + 31) / 32), [256u32, 1, 1]) + } else { + (((m + 31) / 32) * ((n + 31) / 32), [256u32, 1, 1]) + }; + let grid = [nwg, 1, 1]; + + for _ in 0..50 { dev.dispatch_enqueue(kernel, &args_buf, grid, block); } + assert!(dev.submit_wait(30_000), "warmup timeout"); + + let iters = 500; + let t0 = Instant::now(); + for _ in 0..iters { dev.dispatch_enqueue(kernel, &args_buf, grid, block); } + assert!(dev.submit_wait(60_000), "bench timeout"); + t0.elapsed().as_nanos() as f64 / iters as f64 / 1000.0 + }; + + println!(" {:>12} {:>10} {:>10} {:>10} {:>6}", "shape", "TM=128", "TM=32", "best", "pick"); + println!(" {}", "-".repeat(58)); + + for &(m, k, n, label) in shapes { + // Allocate buffers (both W formats) + let w_data = vec![0.001f32; (m * k) as usize]; + let w_row = dev.upload_f32(&w_data).unwrap(); + let w_col = dev.upload_f32_col_major(&w_data, m as usize, k as usize).unwrap(); + let b_buf = dev.upload_f32(&vec![0.0f32; m as usize]).unwrap(); + let x_buf = dev.upload_f32(&vec![0.001f32; (n * k) as usize]).unwrap(); + let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap(); + + let us_128 = bench_kernel(&mut dev, "matmul_blocked", m, k, n, &w_row, &b_buf, &x_buf, &y_buf); + let us_32 = bench_kernel(&mut dev, "matmul_small", m, k, n, &w_col, &b_buf, &x_buf, &y_buf); + + let gf_128 = 2.0 * m as f64 * k as f64 * n as f64 / us_128 / 1e3; + let gf_32 = 2.0 * m as f64 * k as f64 * n as f64 / us_32 / 1e3; + let (best, pick) = if gf_128 >= gf_32 { (gf_128, "TM128") } else { (gf_32, "TM32") }; + + let l = if label.is_empty() { String::new() } else { format!(" {}", label) }; + println!(" {:>5}x{:<5} {:>8.0} {:>8.0} {:>8.0} {:>6}{}", + m, k, gf_128, gf_32, best, pick, l); + } + println!(); +} diff --git a/tests/kfd_matmul_bench_small.rs b/tests/kfd_matmul_bench_small.rs new file mode 100644 index 0000000..ca82d3b --- /dev/null +++ b/tests/kfd_matmul_bench_small.rs @@ -0,0 +1,97 @@ +//! CPU vs GPU crossover benchmark +//! run: cargo test --release --test kfd_matmul_bench_small -- --nocapture --test-threads=1 + +use isis::host::device::kfd::{self, HsaDevice}; +use isis::host::device::kfd::dispatch::{CodeObject, KernArgs}; +use isis::organism::compute::dot; +use std::time::Instant; + +static MATMUL_SMALL_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_small.co"); +static MATMUL_BLOCKED_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co"); + +/// CPU matmul using AVX-512 dot product: Y[j][i] = dot(W[i], X[j]) +fn cpu_matmul(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) { + for j in 0..n { + let x_row = &x[j * k..(j + 1) * k]; + for i in 0..m { + let w_row = &w[i * k..(i + 1) * k]; + y[j * m + i] = dot(w_row, x_row); + } + } +} + +#[test] +fn matmul_bench_small() { + if !kfd::is_available() { eprintln!("skip: no kfd"); return; } + let mut dev = match HsaDevice::open() { Ok(d) => d, Err(e) => { eprintln!("skip: {e}"); return; }}; + + for co_bytes in [MATMUL_SMALL_CO, MATMUL_BLOCKED_CO] { + let co = CodeObject::load(&dev.alloc, co_bytes).unwrap(); + if dev.kernels.is_none() { dev.kernels = Some(std::collections::HashMap::new()); } + for (name, entry) in &co.kernels { dev.kernels.as_mut().unwrap().insert(name.clone(), entry.clone()); } + std::mem::forget(co); + } + + println!(); + println!(" CPU vs GPU crossover (N=32 batch)"); + println!(" {:>10} {:>8} {:>10} {:>10} {:>10} {:>6}", + "shape", "flops", "cpu_us", "gpu_us", "gflop/s", "pick"); + println!(" {}", "-".repeat(62)); + + let shapes: &[(u32, u32)] = &[ + (32, 32), (64, 64), (128, 128), (256, 256), (512, 512), + (1024, 1024), (2048, 2048), (4096, 4096), + ]; + let n = 32u32; + + for &(m, k) in shapes { + let flops = 2.0 * m as f64 * k as f64 * n as f64; + + // --- CPU --- + let w_data = vec![0.001f32; (m * k) as usize]; + let x_data = vec![0.001f32; (n * k) as usize]; + let mut y_cpu = vec![0.0f32; (n * m) as usize]; + cpu_matmul(&w_data, &x_data, &mut y_cpu, m as usize, k as usize, n as usize); + + let iters_cpu = if m <= 256 { 10000 } else if m <= 1024 { 1000 } else { 100 }; + let t0 = Instant::now(); + for _ in 0..iters_cpu { + cpu_matmul(&w_data, &x_data, &mut y_cpu, m as usize, k as usize, n as usize); + } + let cpu_us = t0.elapsed().as_nanos() as f64 / iters_cpu as f64 / 1000.0; + + // --- GPU --- + let (kernel_name, nwg, w_buf) = if m >= 1536 { + ("matmul_blocked", ((m + 127) / 128) * ((n + 31) / 32), + dev.upload_f32(&w_data).unwrap()) + } else { + ("matmul_small", ((m + 31) / 32) * ((n + 31) / 32), + dev.upload_f32_col_major(&w_data, m as usize, k as usize).unwrap()) + }; + let b_buf = dev.upload_f32(&vec![0.0f32; m as usize]).unwrap(); + let x_buf = dev.upload_f32(&x_data).unwrap(); + let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap(); + + let mut args = KernArgs::new(); + args.push_ptr(&w_buf); args.push_ptr(&b_buf); + args.push_ptr(&x_buf); args.push_ptr(&y_buf); + args.push_u32(m); args.push_u32(k); args.push_u32(n); + let args_buf = args.upload(&dev.alloc).unwrap(); + + for _ in 0..50 { dev.dispatch_enqueue(kernel_name, &args_buf, [nwg, 1, 1], [256, 1, 1]); } + assert!(dev.submit_wait(30_000), "warmup timeout"); + + let iters_gpu = 500; + let t0 = Instant::now(); + for _ in 0..iters_gpu { dev.dispatch_enqueue(kernel_name, &args_buf, [nwg, 1, 1], [256, 1, 1]); } + assert!(dev.submit_wait(60_000), "bench timeout"); + let gpu_us = t0.elapsed().as_nanos() as f64 / iters_gpu as f64 / 1000.0; + + let best_gf = flops / gpu_us.min(cpu_us) / 1e3; + let pick = if cpu_us < gpu_us { "CPU" } else { "GPU" }; + + println!(" {:>4}x{:<4} {:>8.0} {:>8.1} {:>8.1} {:>8.0} {:>4}", + m, k, flops, cpu_us, gpu_us, best_gf, pick); + } + println!(); +} diff --git a/tests/kfd_remu.rs b/tests/kfd_remu.rs new file mode 100644 index 0000000..0c4b4ae --- /dev/null +++ b/tests/kfd_remu.rs @@ -0,0 +1,222 @@ +//! GPU kernel validation via remu (RDNA3 ISA emulator). +//! +//! Runs matmul kernels in software before sending to hardware. +//! No GPU required — catches OOB accesses, wrong SGPRs, etc. +//! +//! run: cargo test --test kfd_remu -- --nocapture + +use remu::work_group::WorkGroup; + +/// Extract .text section from an AMDGPU ELF code object (.co). +/// Returns (instructions, code_offset_in_file). +fn extract_text(co: &[u8]) -> Vec { + // Minimal ELF64 parser — just find .text section + assert!(co.len() > 64, "too small for ELF"); + assert!(&co[0..4] == b"\x7fELF", "not ELF"); + assert!(co[4] == 2, "not ELF64"); + + let shoff = u64::from_le_bytes(co[40..48].try_into().unwrap()) as usize; + let shentsize = u16::from_le_bytes(co[58..60].try_into().unwrap()) as usize; + let shnum = u16::from_le_bytes(co[60..62].try_into().unwrap()) as usize; + let shstrndx = u16::from_le_bytes(co[62..64].try_into().unwrap()) as usize; + + // Find string table for section names + let strtab_off = shoff + shstrndx * shentsize; + let str_offset = u64::from_le_bytes(co[strtab_off + 24..strtab_off + 32].try_into().unwrap()) as usize; + + for i in 0..shnum { + let sh = shoff + i * shentsize; + let name_idx = u32::from_le_bytes(co[sh..sh + 4].try_into().unwrap()) as usize; + let name_start = str_offset + name_idx; + let name_end = co[name_start..].iter().position(|&b| b == 0).unwrap() + name_start; + let name = std::str::from_utf8(&co[name_start..name_end]).unwrap_or(""); + + if name == ".text" { + let offset = u64::from_le_bytes(co[sh + 24..sh + 32].try_into().unwrap()) as usize; + let size = u64::from_le_bytes(co[sh + 32..sh + 40].try_into().unwrap()) as usize; + let text = &co[offset..offset + size]; + assert!(size % 4 == 0, ".text not 4-byte aligned"); + return text + .chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect(); + } + } + panic!(".text section not found"); +} + +/// Pack matmul kernargs: W(u64) B(u64) X(u64) Y(u64) M(u32) K(u32) N(u32) +fn pack_kernargs(w: *const f32, b: *const f32, x: *const f32, y: *mut f32, m: u32, k: u32, n: u32) -> Vec { + let mut args = vec![0u64; 6]; // 48 bytes = 6 u64s + args[0] = w as u64; + args[1] = b as u64; + args[2] = x as u64; + args[3] = y as u64; + // M and K packed into one u64 (little-endian: M=low, K=high) + args[4] = (m as u64) | ((k as u64) << 32); + // N in low 32 bits + args[5] = n as u64; + args +} + +fn run_matmul(co: &[u8], w: &[f32], b: &[f32], x: &[f32], y: &mut [f32], m: u32, k: u32, n: u32) { + run_matmul_tiled(co, w, b, x, y, m, k, n, 32, 8); +} + +fn run_matmul_blocked(co: &[u8], w: &[f32], b: &[f32], x: &[f32], y: &mut [f32], m: u32, k: u32, n: u32) { + run_matmul_tiled(co, w, b, x, y, m, k, n, 128, 32); +} + +fn run_matmul_tiled(co: &[u8], w: &[f32], b: &[f32], x: &[f32], y: &mut [f32], m: u32, k: u32, n: u32, tm: u32, tn: u32) { + let kernel = extract_text(co); + let args = pack_kernargs(w.as_ptr(), b.as_ptr(), x.as_ptr(), y.as_mut_ptr(), m, k, n); + + let num_wg_m = (m + tm - 1) / tm; + let num_wg_n = (n + tn - 1) / tn; + let num_wg = num_wg_m * num_wg_n; + + for wg_id in 0..num_wg { + let mut wg = WorkGroup::new(1, [wg_id, 0, 0], [256, 1, 1], &kernel, args.as_ptr()); + wg.exec_waves().expect(&format!("kernel fault in workgroup {wg_id}")); + } +} + +/// Reference matmul: Y = X @ W^T + B +fn reference_matmul(w: &[f32], b: &[f32], x: &[f32], m: usize, k: usize, n: usize) -> Vec { + let mut y = vec![0.0f32; n * m]; + for ni in 0..n { + for mi in 0..m { + let mut sum = b[mi]; + for ki in 0..k { + sum += x[ni * k + ki] * w[mi * k + ki]; + } + y[ni * m + mi] = sum; + } + } + y +} + +#[test] +fn test_matmul_identity_32x32() { + let co = include_bytes!("../src/host/device/kfd/kernels/matmul.co"); + let m = 32u32; + let k = 32u32; + let n = 8u32; + + // W = identity, B = 0.5, X = sequential + let mut w = vec![0.0f32; (m * k) as usize]; + for i in 0..m.min(k) { + w[(i * k + i) as usize] = 1.0; + } + let b = vec![0.5f32; m as usize]; + let x: Vec = (0..n * k).map(|i| i as f32 * 0.1).collect(); + let mut y = vec![0.0f32; (n * m) as usize]; + + run_matmul(co, &w, &b, &x, &mut y, m, k, n); + + let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize); + for i in 0..y.len() { + assert!( + (y[i] - expected[i]).abs() < 1e-3, + "mismatch at {i}: got {}, expected {}", + y[i], + expected[i] + ); + } + println!("PASS: 32x32 batch=8 identity matmul (remu)"); +} + +#[test] +fn test_matmul_random_64x64() { + let co = include_bytes!("../src/host/device/kfd/kernels/matmul.co"); + let m = 64u32; + let k = 64u32; + let n = 8u32; + + // Deterministic pseudo-random + let mut rng = 12345u64; + let mut randf = || -> f32 { + rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1); + ((rng >> 33) as f32 / (1u64 << 31) as f32) - 1.0 + }; + + let w: Vec = (0..m * k).map(|_| randf() * 0.1).collect(); + let b = vec![0.0f32; m as usize]; + let x: Vec = (0..n * k).map(|_| randf()).collect(); + let mut y = vec![0.0f32; (n * m) as usize]; + + run_matmul(co, &w, &b, &x, &mut y, m, k, n); + + let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize); + let max_err = y + .iter() + .zip(expected.iter()) + .map(|(a, b)| (a - b).abs()) + .fold(0.0f32, f32::max); + // FP32 accumulation over K=64 terms can have ~1e-4 relative error + // with values in [-1,1] range, absolute error can reach ~0.01 per term + let max_abs = expected.iter().map(|x| x.abs()).fold(0.0f32, f32::max); + let rel_err = max_err / max_abs.max(1e-6); + println!("64x64 batch=8 max_err={max_err:.6} max_abs={max_abs:.2} rel_err={rel_err:.6}"); + assert!(rel_err < 1e-4, "relative error too large: {rel_err}"); + println!("PASS: 64x64 batch=8 random matmul (remu)"); +} + +// ============ Register-blocked kernel tests ============ + +#[test] +fn test_blocked_identity_128x128() { + let co = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co"); + let m = 128u32; + let k = 8u32; // one tile iteration + let n = 32u32; + + let mut w = vec![0.0f32; (m * k) as usize]; + for i in 0..k { + w[(i * k + i) as usize] = 1.0; + } + let b = vec![0.5f32; m as usize]; + let x: Vec = (0..n * k).map(|i| (i as f32) * 0.01).collect(); + let mut y = vec![0.0f32; (n * m) as usize]; + + run_matmul_blocked(co, &w, &b, &x, &mut y, m, k, n); + + let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize); + let max_err = y.iter().zip(expected.iter()) + .map(|(a, b)| (a - b).abs()) + .fold(0.0f32, f32::max); + println!("blocked 128x8 batch=32: max_err={max_err:.6}"); + assert!(max_err < 1e-3, "max error too large: {max_err}"); + println!("PASS: blocked 128x8 batch=32 identity (remu)"); +} + +#[test] +fn test_blocked_random_256x256() { + let co = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co"); + let m = 256u32; + let k = 64u32; // multiple tile iterations (64/8=8) + let n = 32u32; + + let mut rng = 42u64; + let mut randf = || -> f32 { + rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1); + ((rng >> 33) as f32 / (1u64 << 31) as f32) - 1.0 + }; + + let w: Vec = (0..m * k).map(|_| randf() * 0.1).collect(); + let b = vec![0.0f32; m as usize]; + let x: Vec = (0..n * k).map(|_| randf()).collect(); + let mut y = vec![0.0f32; (n * m) as usize]; + + run_matmul_blocked(co, &w, &b, &x, &mut y, m, k, n); + + let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize); + let max_err = y.iter().zip(expected.iter()) + .map(|(a, b)| (a - b).abs()) + .fold(0.0f32, f32::max); + let max_abs = expected.iter().map(|x| x.abs()).fold(0.0f32, f32::max); + let rel_err = max_err / max_abs.max(1e-6); + println!("blocked 256x64 batch=32: max_err={max_err:.6} rel_err={rel_err:.6}"); + assert!(rel_err < 1e-4, "relative error too large: {rel_err}"); + println!("PASS: blocked 256x64 batch=32 random (remu)"); +}