Initial extraction from isis: direct /dev/kfd GPU compute

AMD RDNA3 GPU driver via raw KFD ioctls. No ROCm, no OpenCL.

- Device discovery and topology queries
- VRAM/GTT/Userptr memory with shared address space
- PM4 compute queue and kernel dispatch
- Hand-written RDNA3 ASM kernels: matmul (3100 GFLOP/s), matvec, superlinear
- Tile-safe buffer padding for OOB protection
- Event-based interrupt wait (no CPU polling)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

tests/kfd_bench.rs

@@ -0,0 +1,160 @@
+//! kfd gpu performance benchmarks.
+//! run: cargo test --release --test kfd_bench -- --nocapture --test-threads=1
+//!
+//! measures gpu dispatch latency vs cpu at various matrix sizes
+//! to find the crossover point where gpu wins.
+
+use isis::host::device::kfd::{self, HsaDevice};
+use isis::host::device::kfd::dispatch::KernArgs;
+use isis::organism::compute::{ComputeBackend, CpuBackend};
+use std::time::Instant;
+
+fn open_gpu() -> Option<std::sync::MutexGuard<'static, HsaDevice>> {
+    use std::sync::{OnceLock, Mutex};
+    static GPU: OnceLock<Option<Mutex<HsaDevice>>> = OnceLock::new();
+    GPU.get_or_init(|| {
+        if !kfd::is_available() { return None; }
+        HsaDevice::open().ok().map(Mutex::new)
+    }).as_ref().map(|m| m.lock().unwrap())
+}
+
+/// cpu reference matvec
+fn cpu_matvec_timed(cpu: &CpuBackend, w: &[f32], b: &[f32], x: &[f32],
+                    y: &mut [f32], out: usize, inp: usize, iters: usize) -> f64 {
+    let t = Instant::now();
+    for _ in 0..iters {
+        cpu.matvec(w, b, x, y, out, inp);
+    }
+    t.elapsed().as_micros() as f64 / iters as f64
+}
+
+/// gpu matvec — allocate once, dispatch N times
+fn gpu_matvec_timed(dev: &mut HsaDevice, w: &[f32], b: &[f32], x: &[f32],
+                    out: u32, inp: u32, iters: usize) -> f64 {
+    let w_buf = dev.upload_f32(w).unwrap();
+    let b_buf = dev.upload_f32(b).unwrap();
+    let x_buf = dev.upload_f32(x).unwrap();
+    let y_buf = dev.alloc.alloc_userptr_public(((out as u64) * 4 + 4095) & !4095).unwrap();
+
+    let mut args = KernArgs::new();
+    args.push_ptr(&w_buf);
+    args.push_ptr(&b_buf);
+    args.push_ptr(&x_buf);
+    args.push_ptr(&y_buf);
+    args.push_u32(out);
+    args.push_u32(inp);
+    let args_buf = args.upload(&dev.alloc).unwrap();
+    let block = out.min(256);
+
+    // warmup
+    dev.dispatch_kernel("matvec", &args_buf, [out, 1, 1], [block, 1, 1]);
+
+    let t = Instant::now();
+    for _ in 0..iters {
+        dev.dispatch_kernel("matvec", &args_buf, [out, 1, 1], [block, 1, 1]);
+    }
+    t.elapsed().as_micros() as f64 / iters as f64
+}
+
+#[test]
+fn bench_matvec_sizes() {
+    let mut dev = match open_gpu() { Some(d) => d, None => {
+        eprintln!("  SKIP: no gpu"); return;
+    }};
+    if dev.kernels.is_none() { eprintln!("  SKIP: no kernels"); return; }
+    let cpu = CpuBackend::new();
+
+    println!();
+    println!("  {:>6} x {:<6}  {:>10}  {:>10}  {:>8}  winner", "out", "in", "cpu (us)", "gpu (us)", "speedup");
+    println!("  {}", "-".repeat(62));
+
+    for &(out, inp) in &[
+        (4, 4), (8, 8), (16, 16), (32, 32), (64, 64),
+        (128, 128), (256, 256), (512, 512),
+        (1024, 1024),
+        (64, 1024), (1024, 64),
+    ] {
+        let n = out * inp;
+        let w: Vec<f32> = (0..n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect();
+        let b: Vec<f32> = (0..out).map(|i| (i % 10) as f32 * 0.1).collect();
+        let x: Vec<f32> = (0..inp).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect();
+        let mut y_cpu = vec![0.0f32; out];
+
+        let iters = if n < 10000 { 1000 } else if n < 100000 { 100 } else { 10 };
+
+        let cpu_us = cpu_matvec_timed(&cpu, &w, &b, &x, &mut y_cpu, out, inp, iters);
+        let gpu_us = gpu_matvec_timed(&mut dev, &w, &b, &x, out as u32, inp as u32, iters);
+
+        let speedup = cpu_us / gpu_us;
+        let winner = if gpu_us < cpu_us { "gpu" } else { "cpu" };
+
+        println!("  {:>6} x {:<6}  {:>10.1}  {:>10.1}  {:>7.2}x  {}",
+            out, inp, cpu_us, gpu_us, speedup, winner);
+    }
+    println!();
+
+    // isis-relevant sizes (from CTM config)
+    println!("  isis workload sizes:");
+    println!("  {:>6} x {:<6}  {:>10}  {:>10}  {:>8}  context", "out", "in", "cpu (us)", "gpu (us)", "speedup");
+    println!("  {}", "-".repeat(72));
+
+    let isis_sizes = [
+        (128, 20, "input synapse (d_input=20 → 128 neurons)"),
+        (256, 128, "attention synapse (128 → 256)"),
+        (256, 256, "output synapse (256 → 256)"),
+        (128, 256, "motor synapse (256 → 128)"),
+        (1024, 20, "large input (d_input=20 → 1024)"),
+        (1024, 1024, "large attention (1024 → 1024)"),
+        (4096, 1024, "xl attention (1024 → 4096)"),
+        (256, 4096, "sync accumulator readout"),
+    ];
+
+    for &(out, inp, label) in &isis_sizes {
+        let n = out * inp;
+        let w: Vec<f32> = (0..n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect();
+        let b: Vec<f32> = (0..out).map(|i| (i % 10) as f32 * 0.1).collect();
+        let x: Vec<f32> = (0..inp).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect();
+        let mut y_cpu = vec![0.0f32; out];
+
+        let iters = if n < 10000 { 1000 } else if n < 100000 { 100 } else { 10 };
+
+        let cpu_us = cpu_matvec_timed(&cpu, &w, &b, &x, &mut y_cpu, out, inp, iters);
+        let gpu_us = gpu_matvec_timed(&mut dev, &w, &b, &x, out as u32, inp as u32, iters);
+
+        let speedup = cpu_us / gpu_us;
+        println!("  {:>6} x {:<6}  {:>10.1}  {:>10.1}  {:>7.2}x  {}",
+            out, inp, cpu_us, gpu_us, speedup, label);
+    }
+    // llm-scale matmul sizes (matvec = batch=1 inference)
+    println!("  llm inference sizes (batch=1 matvec):");
+    println!("  {:>6} x {:<6}  {:>10}  {:>10}  {:>8}  context", "out", "in", "cpu (us)", "gpu (us)", "speedup");
+    println!("  {}", "-".repeat(72));
+
+    let llm_sizes = [
+        (896, 896, "qwen2.5-0.5B hidden"),
+        (4864, 896, "qwen2.5-0.5B mlp up"),
+        (896, 4864, "qwen2.5-0.5B mlp down"),
+        (2048, 2048, "llama-1B hidden"),
+        (5632, 2048, "llama-1B mlp up"),
+    ];
+
+    for &(out, inp, label) in &llm_sizes {
+        let n = out * inp;
+        let w: Vec<f32> = (0..n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect();
+        let b = vec![0.0f32; out];
+        let x: Vec<f32> = (0..inp).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect();
+        let mut y_cpu = vec![0.0f32; out];
+
+        let iters = 10;
+        let cpu_us = cpu_matvec_timed(&cpu, &w, &b, &x, &mut y_cpu, out, inp, iters);
+        let gpu_us = gpu_matvec_timed(&mut dev, &w, &b, &x, out as u32, inp as u32, iters);
+
+        let speedup = cpu_us / gpu_us;
+        let flops = (out * inp * 2) as f64;
+        let gpu_gflops = flops / (gpu_us * 1e3); // GFLOP/s
+        let cpu_gflops = flops / (cpu_us * 1e3);
+        println!("  {:>6} x {:<6}  {:>8.0} ({:>4.1} GF/s)  {:>8.0} ({:>4.1} GF/s)  {:>5.2}x  {}",
+            out, inp, cpu_us, cpu_gflops, gpu_us, gpu_gflops, speedup, label);
+    }
+    println!();
+}

tests/kfd_gpu.rs

@@ -0,0 +1,198 @@
+//! kfd gpu driver integration tests.
+//!
+//! run: cargo test --release --test kfd_gpu -- --nocapture
+//!
+//! these tests require /dev/kfd (amd gpu with kfd driver).
+//! they share a single HsaDevice to avoid acquire_vm conflicts.
+//! each test validates correctness against a cpu reference.
+
+use isis::host::device::kfd::{self, HsaDevice};
+use isis::host::device::kfd::dispatch::KernArgs;
+use std::sync::OnceLock;
+
+/// single shared gpu device for all tests.
+/// tests run sequentially (--test-threads=1 enforced by kfd constraint).
+static GPU: OnceLock<Option<std::sync::Mutex<HsaDevice>>> = OnceLock::new();
+
+fn gpu() -> Option<std::sync::MutexGuard<'static, HsaDevice>> {
+    GPU.get_or_init(|| {
+        if !kfd::is_available() { return None; }
+        HsaDevice::open().ok().map(std::sync::Mutex::new)
+    }).as_ref().map(|m| m.lock().unwrap())
+}
+
+// ─── store kernel ───────────────────────────────────────────
+
+#[test]
+fn store42_writes_correct_values() {
+    let mut dev = match gpu() { Some(d) => d, None => return };
+
+    let n = 32u32;
+    let y = dev.alloc.alloc_userptr_public((n as u64) * 4).unwrap();
+    // write sentinel — must be overwritten
+    y.write_f32(0, &vec![-1.0f32; n as usize]);
+
+    let mut args = KernArgs::new();
+    args.push_ptr(&y);
+    let args_buf = args.upload(&dev.alloc).unwrap();
+
+    assert!(dev.dispatch_kernel("test_store", &args_buf, [n, 1, 1], [n, 1, 1]),
+        "dispatch timed out");
+
+    let result = y.read_f32(0, n as usize);
+    for i in 0..n as usize {
+        assert_eq!(result[i], 42.0, "y[{i}] = {} (sentinel was -1.0)", result[i]);
+    }
+}
+
+// ─── matvec kernel ──────────────────────────────────────────
+
+/// dispatch matvec and return output vector.
+fn gpu_matvec(dev: &mut HsaDevice, w: &[f32], b: &[f32], x: &[f32],
+              out_dim: u32, in_dim: u32) -> Vec<f32> {
+    let w_buf = dev.upload_f32(w).unwrap();
+    let b_buf = dev.upload_f32(b).unwrap();
+    let x_buf = dev.upload_f32(x).unwrap();
+    let y_buf = dev.alloc.alloc_userptr_public(((out_dim as u64) * 4 + 4095) & !4095).unwrap();
+
+    let mut args = KernArgs::new();
+    args.push_ptr(&w_buf);
+    args.push_ptr(&b_buf);
+    args.push_ptr(&x_buf);
+    args.push_ptr(&y_buf);
+    args.push_u32(out_dim);
+    args.push_u32(in_dim);
+    let args_buf = args.upload(&dev.alloc).unwrap();
+
+    let block = out_dim.min(256);
+    assert!(dev.dispatch_kernel("matvec", &args_buf, [out_dim, 1, 1], [block, 1, 1]),
+        "matvec dispatch timed out ({}x{})", out_dim, in_dim);
+
+    y_buf.read_f32(0, out_dim as usize)
+}
+
+/// cpu reference: y = W*x + b
+fn cpu_matvec(w: &[f32], b: &[f32], x: &[f32], out_dim: usize, in_dim: usize) -> Vec<f32> {
+    let mut y = vec![0.0f32; out_dim];
+    for row in 0..out_dim {
+        let mut sum = b[row];
+        for col in 0..in_dim {
+            sum += w[row * in_dim + col] * x[col];
+        }
+        y[row] = sum;
+    }
+    y
+}
+
+fn assert_close(gpu: &[f32], cpu: &[f32], tol: f32, label: &str) {
+    assert_eq!(gpu.len(), cpu.len(), "{label}: length mismatch {} vs {}", gpu.len(), cpu.len());
+    for i in 0..gpu.len() {
+        let err = (gpu[i] - cpu[i]).abs();
+        assert!(err < tol, "{label}: y[{i}] gpu={} cpu={} err={err}", gpu[i], cpu[i]);
+    }
+}
+
+#[test]
+fn matvec_identity_4x4() {
+    let mut dev = match gpu() { Some(d) => d, None => return };
+    let n = 4;
+    let mut w = vec![0.0f32; n * n];
+    for i in 0..n { w[i * n + i] = 1.0; }
+    let b = vec![0.5; n];
+    let x = vec![1.0, 2.0, 3.0, 4.0];
+
+    let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, n as u32, n as u32);
+    let cpu_y = cpu_matvec(&w, &b, &x, n, n);
+    assert_close(&gpu_y, &cpu_y, 1e-5, "identity_4x4");
+}
+
+#[test]
+fn matvec_dense_2x3() {
+    let mut dev = match gpu() { Some(d) => d, None => return };
+    let w = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0f32];
+    let b = vec![0.1, 0.2];
+    let x = vec![1.0, 1.0, 1.0];
+
+    let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, 2, 3);
+    let cpu_y = cpu_matvec(&w, &b, &x, 2, 3);
+    assert_close(&gpu_y, &cpu_y, 1e-4, "dense_2x3");
+}
+
+#[test]
+fn matvec_32x32_vs_cpu() {
+    let mut dev = match gpu() { Some(d) => d, None => return };
+    let n = 32usize;
+
+    // deterministic pseudo-random weights
+    let w: Vec<f32> = (0..n*n).map(|i| ((i * 7 + 3) % 100) as f32 / 100.0 - 0.5).collect();
+    let b: Vec<f32> = (0..n).map(|i| (i % 10) as f32 * 0.1).collect();
+    let x: Vec<f32> = (0..n).map(|i| ((i * 13 + 5) % 100) as f32 / 100.0).collect();
+
+    let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, n as u32, n as u32);
+    let cpu_y = cpu_matvec(&w, &b, &x, n, n);
+    assert_close(&gpu_y, &cpu_y, 1e-3, "random_32x32");
+}
+
+#[test]
+fn matvec_nonsquare_8x16() {
+    let mut dev = match gpu() { Some(d) => d, None => return };
+    let out = 8usize;
+    let inp = 16usize;
+
+    let w = vec![1.0f32; out * inp]; // all ones
+    let b = vec![0.0; out];
+    let x = vec![1.0; inp]; // all ones
+
+    let gpu_y = gpu_matvec(&mut dev, &w, &b, &x, out as u32, inp as u32);
+    let cpu_y = cpu_matvec(&w, &b, &x, out, inp);
+    assert_close(&gpu_y, &cpu_y, 1e-3, "nonsquare_8x16");
+
+    // each row should sum to 16.0 (16 ones)
+    for i in 0..out {
+        assert!((gpu_y[i] - 16.0).abs() < 1e-3, "y[{i}] = {}", gpu_y[i]);
+    }
+}
+
+#[test]
+fn matvec_scalar_1x1() {
+    let mut dev = match gpu() { Some(d) => d, None => return };
+    let gpu_y = gpu_matvec(&mut dev, &[3.0], &[0.5], &[2.0], 1, 1);
+    assert!((gpu_y[0] - 6.5).abs() < 1e-5, "1x1: {} expected 6.5", gpu_y[0]);
+}
+
+// ─── async future ───────────────────────────────────────────
+
+#[test]
+fn dispatch_async_returns_future() {
+    use isis::host::device::kfd::compute::GpuFuture;
+
+    let mut dev = match gpu() { Some(d) => d, None => return };
+
+    let y = dev.alloc.alloc_userptr_public(4096).unwrap();
+    let mut args = KernArgs::new();
+    args.push_ptr(&y);
+    let args_buf = args.upload(&dev.alloc).unwrap();
+
+    let future = dev.dispatch_async("test_store", &args_buf, [4, 1, 1], [4, 1, 1]);
+    assert!(future.is_some(), "dispatch_async returned None");
+
+    let future = future.unwrap();
+    assert!(!future.poll() || true, "poll should not panic"); // may already be done
+    let elapsed = future.wait(1_000_000);
+    assert!(elapsed.is_some(), "future timed out after 1s");
+
+    let result = y.read_f32(0, 4);
+    assert_eq!(result, vec![42.0; 4]);
+}
+
+#[test]
+fn dispatch_nonexistent_kernel_returns_none() {
+    let mut dev = match gpu() { Some(d) => d, None => return };
+    let y = dev.alloc.alloc_userptr_public(4096).unwrap();
+    let mut args = KernArgs::new();
+    args.push_ptr(&y);
+    let args_buf = args.upload(&dev.alloc).unwrap();
+
+    let future = dev.dispatch_async("nonexistent_kernel", &args_buf, [1, 1, 1], [1, 1, 1]);
+    assert!(future.is_none(), "should return None for unknown kernel");
+}

tests/kfd_matmul_bench.rs

@@ -0,0 +1,167 @@
+//! matmul kernel benchmark — pure rust dispatch.
+//! run: cargo test --release --test kfd_matmul_bench -- --nocapture --test-threads=1
+
+use isis::host::device::kfd::{self, HsaDevice};
+use isis::host::device::kfd::dispatch::{CodeObject, KernArgs};
+use isis::host::device::kfd::memory::GpuBuffer;
+use std::time::Instant;
+
+static MATMUL_BLOCKED_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co");
+static MATMUL_SMALL_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_small.co");
+
+#[test]
+fn matmul_bench() {
+    if !kfd::is_available() { eprintln!("skip: no kfd"); return; }
+    let mut dev = match HsaDevice::open() { Ok(d) => d, Err(e) => { eprintln!("skip: {e}"); return; }};
+
+    // Load both matmul kernels
+    for co_bytes in [MATMUL_BLOCKED_CO, MATMUL_SMALL_CO] {
+        let co = CodeObject::load(&dev.alloc, co_bytes).unwrap();
+        if dev.kernels.is_none() { dev.kernels = Some(std::collections::HashMap::new()); }
+        for (name, entry) in &co.kernels {
+            dev.kernels.as_mut().unwrap().insert(name.clone(), entry.clone());
+        }
+        std::mem::forget(co); // keep code in VRAM
+    }
+
+    if let Some(mhz) = dev.current_sclk_mhz() {
+        println!("  sclk: {} mhz", mhz);
+    }
+    println!();
+
+    // ---- Correctness check for matmul_blocked (TM=128) ----
+    {
+        let m = 128u32; let k = 8u32; let n = 32u32;
+        let mut w_data = vec![0.0f32; (m * k) as usize];
+        for i in 0..m as usize { w_data[i * k as usize] = (i + 1) as f32; }
+        let b_data = vec![0.0f32; m as usize];
+        let mut x_data = vec![0.0f32; (n * k) as usize];
+        for j in 0..n as usize { x_data[j * k as usize] = 1.0; }
+
+        let w_buf = dev.upload_f32(&w_data).unwrap();
+        let b_buf = dev.upload_f32(&b_data).unwrap();
+        let x_buf = dev.upload_f32(&x_data).unwrap();
+        let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap();
+
+        let mut args = KernArgs::new();
+        args.push_ptr(&w_buf); args.push_ptr(&b_buf);
+        args.push_ptr(&x_buf); args.push_ptr(&y_buf);
+        args.push_u32(m); args.push_u32(k); args.push_u32(n);
+        let args_buf = args.upload(&dev.alloc).unwrap();
+
+        let nwg = ((m + 127) / 128) * ((n + 31) / 32);
+        dev.dispatch_kernel("matmul_blocked", &args_buf, [nwg, 1, 1], [256, 1, 1]);
+
+        let y_slice = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr as *const f32, (n * m) as usize) };
+        print!("  matmul_blocked (TM=128): ");
+        let mut ok = true;
+        for j in 0..n as usize {
+            for i in 0..m as usize {
+                let expected = (i + 1) as f32;
+                if (y_slice[j * m as usize + i] - expected).abs() > 0.1 { ok = false; break; }
+            }
+            if !ok { break; }
+        }
+        println!("{}", if ok { "PASS" } else { "FAIL" });
+        assert!(ok, "matmul_blocked correctness failed");
+    }
+
+    // ---- Correctness check for matmul_small (TM=32) ----
+    {
+        let m = 32u32; let k = 8u32; let n = 32u32;
+        let mut w_data = vec![0.0f32; (m * k) as usize];
+        for i in 0..m as usize { w_data[i * k as usize] = (i + 1) as f32; }
+        let b_data = vec![0.0f32; m as usize];
+        let mut x_data = vec![0.0f32; (n * k) as usize];
+        for j in 0..n as usize { x_data[j * k as usize] = 1.0; }
+
+        let w_buf = dev.upload_f32_col_major(&w_data, m as usize, k as usize).unwrap();
+        let b_buf = dev.upload_f32(&b_data).unwrap();
+        let x_buf = dev.upload_f32(&x_data).unwrap();
+        let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap();
+
+        let mut args = KernArgs::new();
+        args.push_ptr(&w_buf); args.push_ptr(&b_buf);
+        args.push_ptr(&x_buf); args.push_ptr(&y_buf);
+        args.push_u32(m); args.push_u32(k); args.push_u32(n);
+        let args_buf = args.upload(&dev.alloc).unwrap();
+
+        let nwg = ((m + 31) / 32) * ((n + 31) / 32);
+        dev.dispatch_kernel("matmul_small", &args_buf, [nwg, 1, 1], [256, 1, 1]);
+
+        let y_slice = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr as *const f32, (n * m) as usize) };
+        print!("  matmul_small   (TM=32):  ");
+        let mut ok = true;
+        for j in 0..n as usize {
+            for i in 0..m as usize {
+                let expected = (i + 1) as f32;
+                if (y_slice[j * m as usize + i] - expected).abs() > 0.1 { ok = false; break; }
+            }
+            if !ok { break; }
+        }
+        println!("{}", if ok { "PASS" } else { "FAIL" });
+        assert!(ok, "matmul_small correctness failed");
+    }
+    println!();
+
+    // ---- Performance benchmark: both kernels + dispatch selector ----
+    let shapes: &[(u32, u32, u32, &str)] = &[
+        (512,  512,  32, ""),
+        (1024, 1024, 32, ""),
+        (2048, 2048, 32, ""),
+        (4096, 4096, 32, "qwen attn"),
+        (4096, 11008, 32, "qwen mlp"),
+        (4864, 896,  32, "isis layer"),
+    ];
+
+    // bench helper
+    let bench_kernel = |dev: &mut HsaDevice, kernel: &str, m: u32, k: u32, n: u32,
+                        w_buf: &GpuBuffer, b_buf: &GpuBuffer, x_buf: &GpuBuffer, y_buf: &GpuBuffer| -> f64 {
+        let mut args = KernArgs::new();
+        args.push_ptr(w_buf); args.push_ptr(b_buf);
+        args.push_ptr(x_buf); args.push_ptr(y_buf);
+        args.push_u32(m); args.push_u32(k); args.push_u32(n);
+        let args_buf = args.upload(&dev.alloc).unwrap();
+
+        let (nwg, block) = if kernel == "matmul_blocked" {
+            (((m + 127) / 128) * ((n + 31) / 32), [256u32, 1, 1])
+        } else {
+            (((m + 31) / 32) * ((n + 31) / 32), [256u32, 1, 1])
+        };
+        let grid = [nwg, 1, 1];
+
+        for _ in 0..50 { dev.dispatch_enqueue(kernel, &args_buf, grid, block); }
+        assert!(dev.submit_wait(30_000), "warmup timeout");
+
+        let iters = 500;
+        let t0 = Instant::now();
+        for _ in 0..iters { dev.dispatch_enqueue(kernel, &args_buf, grid, block); }
+        assert!(dev.submit_wait(60_000), "bench timeout");
+        t0.elapsed().as_nanos() as f64 / iters as f64 / 1000.0
+    };
+
+    println!("  {:>12}  {:>10}  {:>10}  {:>10}  {:>6}", "shape", "TM=128", "TM=32", "best", "pick");
+    println!("  {}", "-".repeat(58));
+
+    for &(m, k, n, label) in shapes {
+        // Allocate buffers (both W formats)
+        let w_data = vec![0.001f32; (m * k) as usize];
+        let w_row = dev.upload_f32(&w_data).unwrap();
+        let w_col = dev.upload_f32_col_major(&w_data, m as usize, k as usize).unwrap();
+        let b_buf = dev.upload_f32(&vec![0.0f32; m as usize]).unwrap();
+        let x_buf = dev.upload_f32(&vec![0.001f32; (n * k) as usize]).unwrap();
+        let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap();
+
+        let us_128 = bench_kernel(&mut dev, "matmul_blocked", m, k, n, &w_row, &b_buf, &x_buf, &y_buf);
+        let us_32  = bench_kernel(&mut dev, "matmul_small",   m, k, n, &w_col, &b_buf, &x_buf, &y_buf);
+
+        let gf_128 = 2.0 * m as f64 * k as f64 * n as f64 / us_128 / 1e3;
+        let gf_32  = 2.0 * m as f64 * k as f64 * n as f64 / us_32  / 1e3;
+        let (best, pick) = if gf_128 >= gf_32 { (gf_128, "TM128") } else { (gf_32, "TM32") };
+
+        let l = if label.is_empty() { String::new() } else { format!("  {}", label) };
+        println!("  {:>5}x{:<5}  {:>8.0}  {:>8.0}  {:>8.0}  {:>6}{}",
+            m, k, gf_128, gf_32, best, pick, l);
+    }
+    println!();
+}

tests/kfd_matmul_bench_small.rs

@@ -0,0 +1,97 @@
+//! CPU vs GPU crossover benchmark
+//! run: cargo test --release --test kfd_matmul_bench_small -- --nocapture --test-threads=1
+
+use isis::host::device::kfd::{self, HsaDevice};
+use isis::host::device::kfd::dispatch::{CodeObject, KernArgs};
+use isis::organism::compute::dot;
+use std::time::Instant;
+
+static MATMUL_SMALL_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_small.co");
+static MATMUL_BLOCKED_CO: &[u8] = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co");
+
+/// CPU matmul using AVX-512 dot product: Y[j][i] = dot(W[i], X[j])
+fn cpu_matmul(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) {
+    for j in 0..n {
+        let x_row = &x[j * k..(j + 1) * k];
+        for i in 0..m {
+            let w_row = &w[i * k..(i + 1) * k];
+            y[j * m + i] = dot(w_row, x_row);
+        }
+    }
+}
+
+#[test]
+fn matmul_bench_small() {
+    if !kfd::is_available() { eprintln!("skip: no kfd"); return; }
+    let mut dev = match HsaDevice::open() { Ok(d) => d, Err(e) => { eprintln!("skip: {e}"); return; }};
+
+    for co_bytes in [MATMUL_SMALL_CO, MATMUL_BLOCKED_CO] {
+        let co = CodeObject::load(&dev.alloc, co_bytes).unwrap();
+        if dev.kernels.is_none() { dev.kernels = Some(std::collections::HashMap::new()); }
+        for (name, entry) in &co.kernels { dev.kernels.as_mut().unwrap().insert(name.clone(), entry.clone()); }
+        std::mem::forget(co);
+    }
+
+    println!();
+    println!("  CPU vs GPU crossover (N=32 batch)");
+    println!("  {:>10} {:>8} {:>10} {:>10} {:>10} {:>6}",
+        "shape", "flops", "cpu_us", "gpu_us", "gflop/s", "pick");
+    println!("  {}", "-".repeat(62));
+
+    let shapes: &[(u32, u32)] = &[
+        (32, 32), (64, 64), (128, 128), (256, 256), (512, 512),
+        (1024, 1024), (2048, 2048), (4096, 4096),
+    ];
+    let n = 32u32;
+
+    for &(m, k) in shapes {
+        let flops = 2.0 * m as f64 * k as f64 * n as f64;
+
+        // --- CPU ---
+        let w_data = vec![0.001f32; (m * k) as usize];
+        let x_data = vec![0.001f32; (n * k) as usize];
+        let mut y_cpu = vec![0.0f32; (n * m) as usize];
+        cpu_matmul(&w_data, &x_data, &mut y_cpu, m as usize, k as usize, n as usize);
+
+        let iters_cpu = if m <= 256 { 10000 } else if m <= 1024 { 1000 } else { 100 };
+        let t0 = Instant::now();
+        for _ in 0..iters_cpu {
+            cpu_matmul(&w_data, &x_data, &mut y_cpu, m as usize, k as usize, n as usize);
+        }
+        let cpu_us = t0.elapsed().as_nanos() as f64 / iters_cpu as f64 / 1000.0;
+
+        // --- GPU ---
+        let (kernel_name, nwg, w_buf) = if m >= 1536 {
+            ("matmul_blocked", ((m + 127) / 128) * ((n + 31) / 32),
+             dev.upload_f32(&w_data).unwrap())
+        } else {
+            ("matmul_small", ((m + 31) / 32) * ((n + 31) / 32),
+             dev.upload_f32_col_major(&w_data, m as usize, k as usize).unwrap())
+        };
+        let b_buf = dev.upload_f32(&vec![0.0f32; m as usize]).unwrap();
+        let x_buf = dev.upload_f32(&x_data).unwrap();
+        let y_buf = dev.alloc_output((n as usize * m as usize * 4 + 64) as usize).unwrap();
+
+        let mut args = KernArgs::new();
+        args.push_ptr(&w_buf); args.push_ptr(&b_buf);
+        args.push_ptr(&x_buf); args.push_ptr(&y_buf);
+        args.push_u32(m); args.push_u32(k); args.push_u32(n);
+        let args_buf = args.upload(&dev.alloc).unwrap();
+
+        for _ in 0..50 { dev.dispatch_enqueue(kernel_name, &args_buf, [nwg, 1, 1], [256, 1, 1]); }
+        assert!(dev.submit_wait(30_000), "warmup timeout");
+
+        let iters_gpu = 500;
+        let t0 = Instant::now();
+        for _ in 0..iters_gpu { dev.dispatch_enqueue(kernel_name, &args_buf, [nwg, 1, 1], [256, 1, 1]); }
+        assert!(dev.submit_wait(60_000), "bench timeout");
+        let gpu_us = t0.elapsed().as_nanos() as f64 / iters_gpu as f64 / 1000.0;
+
+        let best_gf = flops / gpu_us.min(cpu_us) / 1e3;
+        let pick = if cpu_us < gpu_us { "CPU" } else { "GPU" };
+
+        println!("  {:>4}x{:<4} {:>8.0} {:>8.1} {:>8.1} {:>8.0}   {:>4}",
+            m, k, flops, cpu_us, gpu_us, best_gf, pick);
+    }
+    println!();
+}

tests/kfd_remu.rs

@@ -0,0 +1,222 @@
+//! GPU kernel validation via remu (RDNA3 ISA emulator).
+//!
+//! Runs matmul kernels in software before sending to hardware.
+//! No GPU required — catches OOB accesses, wrong SGPRs, etc.
+//!
+//! run: cargo test --test kfd_remu -- --nocapture
+
+use remu::work_group::WorkGroup;
+
+/// Extract .text section from an AMDGPU ELF code object (.co).
+/// Returns (instructions, code_offset_in_file).
+fn extract_text(co: &[u8]) -> Vec<u32> {
+    // Minimal ELF64 parser — just find .text section
+    assert!(co.len() > 64, "too small for ELF");
+    assert!(&co[0..4] == b"\x7fELF", "not ELF");
+    assert!(co[4] == 2, "not ELF64");
+
+    let shoff = u64::from_le_bytes(co[40..48].try_into().unwrap()) as usize;
+    let shentsize = u16::from_le_bytes(co[58..60].try_into().unwrap()) as usize;
+    let shnum = u16::from_le_bytes(co[60..62].try_into().unwrap()) as usize;
+    let shstrndx = u16::from_le_bytes(co[62..64].try_into().unwrap()) as usize;
+
+    // Find string table for section names
+    let strtab_off = shoff + shstrndx * shentsize;
+    let str_offset = u64::from_le_bytes(co[strtab_off + 24..strtab_off + 32].try_into().unwrap()) as usize;
+
+    for i in 0..shnum {
+        let sh = shoff + i * shentsize;
+        let name_idx = u32::from_le_bytes(co[sh..sh + 4].try_into().unwrap()) as usize;
+        let name_start = str_offset + name_idx;
+        let name_end = co[name_start..].iter().position(|&b| b == 0).unwrap() + name_start;
+        let name = std::str::from_utf8(&co[name_start..name_end]).unwrap_or("");
+
+        if name == ".text" {
+            let offset = u64::from_le_bytes(co[sh + 24..sh + 32].try_into().unwrap()) as usize;
+            let size = u64::from_le_bytes(co[sh + 32..sh + 40].try_into().unwrap()) as usize;
+            let text = &co[offset..offset + size];
+            assert!(size % 4 == 0, ".text not 4-byte aligned");
+            return text
+                .chunks_exact(4)
+                .map(|c| u32::from_le_bytes(c.try_into().unwrap()))
+                .collect();
+        }
+    }
+    panic!(".text section not found");
+}
+
+/// Pack matmul kernargs: W(u64) B(u64) X(u64) Y(u64) M(u32) K(u32) N(u32)
+fn pack_kernargs(w: *const f32, b: *const f32, x: *const f32, y: *mut f32, m: u32, k: u32, n: u32) -> Vec<u64> {
+    let mut args = vec![0u64; 6]; // 48 bytes = 6 u64s
+    args[0] = w as u64;
+    args[1] = b as u64;
+    args[2] = x as u64;
+    args[3] = y as u64;
+    // M and K packed into one u64 (little-endian: M=low, K=high)
+    args[4] = (m as u64) | ((k as u64) << 32);
+    // N in low 32 bits
+    args[5] = n as u64;
+    args
+}
+
+fn run_matmul(co: &[u8], w: &[f32], b: &[f32], x: &[f32], y: &mut [f32], m: u32, k: u32, n: u32) {
+    run_matmul_tiled(co, w, b, x, y, m, k, n, 32, 8);
+}
+
+fn run_matmul_blocked(co: &[u8], w: &[f32], b: &[f32], x: &[f32], y: &mut [f32], m: u32, k: u32, n: u32) {
+    run_matmul_tiled(co, w, b, x, y, m, k, n, 128, 32);
+}
+
+fn run_matmul_tiled(co: &[u8], w: &[f32], b: &[f32], x: &[f32], y: &mut [f32], m: u32, k: u32, n: u32, tm: u32, tn: u32) {
+    let kernel = extract_text(co);
+    let args = pack_kernargs(w.as_ptr(), b.as_ptr(), x.as_ptr(), y.as_mut_ptr(), m, k, n);
+
+    let num_wg_m = (m + tm - 1) / tm;
+    let num_wg_n = (n + tn - 1) / tn;
+    let num_wg = num_wg_m * num_wg_n;
+
+    for wg_id in 0..num_wg {
+        let mut wg = WorkGroup::new(1, [wg_id, 0, 0], [256, 1, 1], &kernel, args.as_ptr());
+        wg.exec_waves().expect(&format!("kernel fault in workgroup {wg_id}"));
+    }
+}
+
+/// Reference matmul: Y = X @ W^T + B
+fn reference_matmul(w: &[f32], b: &[f32], x: &[f32], m: usize, k: usize, n: usize) -> Vec<f32> {
+    let mut y = vec![0.0f32; n * m];
+    for ni in 0..n {
+        for mi in 0..m {
+            let mut sum = b[mi];
+            for ki in 0..k {
+                sum += x[ni * k + ki] * w[mi * k + ki];
+            }
+            y[ni * m + mi] = sum;
+        }
+    }
+    y
+}
+
+#[test]
+fn test_matmul_identity_32x32() {
+    let co = include_bytes!("../src/host/device/kfd/kernels/matmul.co");
+    let m = 32u32;
+    let k = 32u32;
+    let n = 8u32;
+
+    // W = identity, B = 0.5, X = sequential
+    let mut w = vec![0.0f32; (m * k) as usize];
+    for i in 0..m.min(k) {
+        w[(i * k + i) as usize] = 1.0;
+    }
+    let b = vec![0.5f32; m as usize];
+    let x: Vec<f32> = (0..n * k).map(|i| i as f32 * 0.1).collect();
+    let mut y = vec![0.0f32; (n * m) as usize];
+
+    run_matmul(co, &w, &b, &x, &mut y, m, k, n);
+
+    let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize);
+    for i in 0..y.len() {
+        assert!(
+            (y[i] - expected[i]).abs() < 1e-3,
+            "mismatch at {i}: got {}, expected {}",
+            y[i],
+            expected[i]
+        );
+    }
+    println!("PASS: 32x32 batch=8 identity matmul (remu)");
+}
+
+#[test]
+fn test_matmul_random_64x64() {
+    let co = include_bytes!("../src/host/device/kfd/kernels/matmul.co");
+    let m = 64u32;
+    let k = 64u32;
+    let n = 8u32;
+
+    // Deterministic pseudo-random
+    let mut rng = 12345u64;
+    let mut randf = || -> f32 {
+        rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1);
+        ((rng >> 33) as f32 / (1u64 << 31) as f32) - 1.0
+    };
+
+    let w: Vec<f32> = (0..m * k).map(|_| randf() * 0.1).collect();
+    let b = vec![0.0f32; m as usize];
+    let x: Vec<f32> = (0..n * k).map(|_| randf()).collect();
+    let mut y = vec![0.0f32; (n * m) as usize];
+
+    run_matmul(co, &w, &b, &x, &mut y, m, k, n);
+
+    let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize);
+    let max_err = y
+        .iter()
+        .zip(expected.iter())
+        .map(|(a, b)| (a - b).abs())
+        .fold(0.0f32, f32::max);
+    // FP32 accumulation over K=64 terms can have ~1e-4 relative error
+    // with values in [-1,1] range, absolute error can reach ~0.01 per term
+    let max_abs = expected.iter().map(|x| x.abs()).fold(0.0f32, f32::max);
+    let rel_err = max_err / max_abs.max(1e-6);
+    println!("64x64 batch=8 max_err={max_err:.6} max_abs={max_abs:.2} rel_err={rel_err:.6}");
+    assert!(rel_err < 1e-4, "relative error too large: {rel_err}");
+    println!("PASS: 64x64 batch=8 random matmul (remu)");
+}
+
+// ============ Register-blocked kernel tests ============
+
+#[test]
+fn test_blocked_identity_128x128() {
+    let co = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co");
+    let m = 128u32;
+    let k = 8u32; // one tile iteration
+    let n = 32u32;
+
+    let mut w = vec![0.0f32; (m * k) as usize];
+    for i in 0..k {
+        w[(i * k + i) as usize] = 1.0;
+    }
+    let b = vec![0.5f32; m as usize];
+    let x: Vec<f32> = (0..n * k).map(|i| (i as f32) * 0.01).collect();
+    let mut y = vec![0.0f32; (n * m) as usize];
+
+    run_matmul_blocked(co, &w, &b, &x, &mut y, m, k, n);
+
+    let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize);
+    let max_err = y.iter().zip(expected.iter())
+        .map(|(a, b)| (a - b).abs())
+        .fold(0.0f32, f32::max);
+    println!("blocked 128x8 batch=32: max_err={max_err:.6}");
+    assert!(max_err < 1e-3, "max error too large: {max_err}");
+    println!("PASS: blocked 128x8 batch=32 identity (remu)");
+}
+
+#[test]
+fn test_blocked_random_256x256() {
+    let co = include_bytes!("../src/host/device/kfd/kernels/matmul_blocked.co");
+    let m = 256u32;
+    let k = 64u32; // multiple tile iterations (64/8=8)
+    let n = 32u32;
+
+    let mut rng = 42u64;
+    let mut randf = || -> f32 {
+        rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1);
+        ((rng >> 33) as f32 / (1u64 << 31) as f32) - 1.0
+    };
+
+    let w: Vec<f32> = (0..m * k).map(|_| randf() * 0.1).collect();
+    let b = vec![0.0f32; m as usize];
+    let x: Vec<f32> = (0..n * k).map(|_| randf()).collect();
+    let mut y = vec![0.0f32; (n * m) as usize];
+
+    run_matmul_blocked(co, &w, &b, &x, &mut y, m, k, n);
+
+    let expected = reference_matmul(&w, &b, &x, m as usize, k as usize, n as usize);
+    let max_err = y.iter().zip(expected.iter())
+        .map(|(a, b)| (a - b).abs())
+        .fold(0.0f32, f32::max);
+    let max_abs = expected.iter().map(|x| x.abs()).fold(0.0f32, f32::max);
+    let rel_err = max_err / max_abs.max(1e-6);
+    println!("blocked 256x64 batch=32: max_err={max_err:.6} rel_err={rel_err:.6}");
+    assert!(rel_err < 1e-4, "relative error too large: {rel_err}");
+    println!("PASS: blocked 256x64 batch=32 random (remu)");
+}