use crate::api::icd::*; use crate::core::context::*; use crate::core::device::*; use crate::core::kernel::*; use crate::core::platform::Platform; use crate::impl_cl_type_trait; use mesa_rust::compiler::clc::spirv::SPIRVBin; use mesa_rust::compiler::clc::*; use mesa_rust::compiler::nir::*; use mesa_rust::util::disk_cache::*; use mesa_rust_gen::*; use rusticl_opencl_gen::*; use std::collections::HashMap; use std::collections::HashSet; use std::ffi::CString; use std::mem::size_of; use std::ptr; use std::slice; use std::sync::atomic::AtomicU32; use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::sync::MutexGuard; use std::sync::Once; const BIN_HEADER_SIZE_V1: usize = // 1. format version size_of::() + // 2. spirv len size_of::() + // 3. binary_type size_of::(); const BIN_HEADER_SIZE: usize = BIN_HEADER_SIZE_V1; // kernel cache static mut DISK_CACHE: Option = None; static DISK_CACHE_ONCE: Once = Once::new(); fn get_disk_cache() -> &'static Option { unsafe { DISK_CACHE_ONCE.call_once(|| { DISK_CACHE = DiskCache::new("rusticl", "rusticl", 0); }); &DISK_CACHE } } pub enum ProgramSourceType { Binary, Linked, Src(CString), Il(spirv::SPIRVBin), } #[repr(C)] pub struct Program { pub base: CLObjectBase, pub context: Arc, pub devs: Vec>, pub src: ProgramSourceType, pub kernel_count: AtomicU32, build: Mutex, } impl_cl_type_trait!(cl_program, Program, CL_INVALID_PROGRAM); #[derive(Clone)] pub struct NirKernelBuild { pub nirs: HashMap, Arc>, pub args: Vec, pub internal_args: Vec, pub attributes_string: String, } pub(super) struct ProgramBuild { builds: HashMap, ProgramDevBuild>, spec_constants: HashMap, kernels: Vec, kernel_builds: HashMap, } impl ProgramBuild { fn attribute_str(&self, kernel: &str, d: &Arc) -> String { let info = self.dev_build(d); let attributes_strings = [ info.spirv.as_ref().unwrap().vec_type_hint(kernel), info.spirv.as_ref().unwrap().local_size(kernel), info.spirv.as_ref().unwrap().local_size_hint(kernel), ]; let attributes_strings: Vec<_> = attributes_strings .iter() .flatten() .map(String::as_str) .collect(); attributes_strings.join(",") } fn args(&self, dev: &Arc, kernel: &str) -> Vec { self.dev_build(dev).spirv.as_ref().unwrap().args(kernel) } fn build_nirs(&mut self, is_src: bool) { for kernel_name in &self.kernels { let kernel_args: HashSet<_> = self .devs_with_build() .iter() .map(|d| self.args(d, kernel_name)) .collect(); let args = kernel_args.into_iter().next().unwrap(); let mut nirs = HashMap::new(); let mut args_set = HashSet::new(); let mut internal_args_set = HashSet::new(); let mut attributes_string_set = HashSet::new(); // TODO: we could run this in parallel? for d in self.devs_with_build() { let (nir, args, internal_args) = convert_spirv_to_nir(self, kernel_name, &args, d); let attributes_string = self.attribute_str(kernel_name, d); nirs.insert(d.clone(), Arc::new(nir)); args_set.insert(args); internal_args_set.insert(internal_args); attributes_string_set.insert(attributes_string); } // we want the same (internal) args for every compiled kernel, for now assert!(args_set.len() == 1); assert!(internal_args_set.len() == 1); assert!(attributes_string_set.len() == 1); let args = args_set.into_iter().next().unwrap(); let internal_args = internal_args_set.into_iter().next().unwrap(); // spec: For kernels not created from OpenCL C source and the clCreateProgramWithSource // API call the string returned from this query [CL_KERNEL_ATTRIBUTES] will be empty. let attributes_string = if is_src { attributes_string_set.into_iter().next().unwrap() } else { String::new() }; self.kernel_builds.insert( kernel_name.clone(), NirKernelBuild { nirs: nirs, args: args, internal_args: internal_args, attributes_string: attributes_string, }, ); } } fn dev_build(&self, dev: &Device) -> &ProgramDevBuild { self.builds.get(dev).unwrap() } fn dev_build_mut(&mut self, dev: &Device) -> &mut ProgramDevBuild { self.builds.get_mut(dev).unwrap() } fn devs_with_build(&self) -> Vec<&Arc> { self.builds .iter() .filter(|(_, build)| build.status == CL_BUILD_SUCCESS as cl_build_status) .map(|(d, _)| d) .collect() } pub fn hash_key(&self, dev: &Arc, name: &str) -> Option { if let Some(cache) = dev.screen().shader_cache() { let info = self.dev_build(dev); assert_eq!(info.status, CL_BUILD_SUCCESS as cl_build_status); let spirv = info.spirv.as_ref().unwrap(); let mut bin = spirv.to_bin().to_vec(); bin.extend_from_slice(name.as_bytes()); for (k, v) in &self.spec_constants { bin.extend_from_slice(&k.to_ne_bytes()); unsafe { // SAFETY: we fully initialize this union bin.extend_from_slice(&v.u64_.to_ne_bytes()); } } Some(cache.gen_key(&bin)) } else { None } } pub fn to_nir(&self, kernel: &str, d: &Arc) -> NirShader { let mut spec_constants: Vec<_> = self .spec_constants .iter() .map(|(&id, &value)| nir_spirv_specialization { id: id, value: value, defined_on_module: true, }) .collect(); let info = self.dev_build(d); assert_eq!(info.status, CL_BUILD_SUCCESS as cl_build_status); let mut log = Platform::dbg().program.then(Vec::new); let nir = info.spirv.as_ref().unwrap().to_nir( kernel, d.screen .nir_shader_compiler_options(pipe_shader_type::PIPE_SHADER_COMPUTE), &d.lib_clc, &mut spec_constants, d.address_bits(), log.as_mut(), ); if let Some(log) = log { for line in log { eprintln!("{}", line); } }; nir.unwrap() } } struct ProgramDevBuild { spirv: Option, status: cl_build_status, options: String, log: String, bin_type: cl_program_binary_type, } fn prepare_options(options: &str, dev: &Device) -> Vec { let mut options = options.to_owned(); if !options.contains("-cl-std=CL") { options.push_str(" -cl-std=CL"); options.push_str(dev.clc_version.api_str()); } if !dev.image_supported() { options.push_str(" -U__IMAGE_SUPPORT__"); } options.push_str(" -D__OPENCL_VERSION__="); options.push_str(dev.cl_version.clc_str()); let mut res = Vec::new(); // we seperate on a ' ' unless we hit a " let mut sep = ' '; let mut old = 0; for (i, c) in options.char_indices() { if c == '"' { if sep == ' ' { sep = '"'; } else { sep = ' '; } } if c == '"' || c == sep { // beware of double seps if old != i { res.push(&options[old..i]); } old = i + c.len_utf8(); } } // add end of the string res.push(&options[old..]); res.iter() .map(|&a| match a { "-cl-denorms-are-zero" => "-fdenormal-fp-math=positive-zero", _ => a, }) .map(CString::new) .map(Result::unwrap) .collect() } impl Program { fn create_default_builds(devs: &[Arc]) -> HashMap, ProgramDevBuild> { devs.iter() .map(|d| { ( d.clone(), ProgramDevBuild { spirv: None, status: CL_BUILD_NONE, log: String::from(""), options: String::from(""), bin_type: CL_PROGRAM_BINARY_TYPE_NONE, }, ) }) .collect() } pub fn new(context: &Arc, devs: &[Arc], src: CString) -> Arc { Arc::new(Self { base: CLObjectBase::new(), context: context.clone(), devs: devs.to_vec(), src: ProgramSourceType::Src(src), kernel_count: AtomicU32::new(0), build: Mutex::new(ProgramBuild { builds: Self::create_default_builds(devs), spec_constants: HashMap::new(), kernels: Vec::new(), kernel_builds: HashMap::new(), }), }) } pub fn from_bins( context: Arc, devs: Vec>, bins: &[&[u8]], ) -> Arc { let mut builds = HashMap::new(); let mut kernels = HashSet::new(); for (d, b) in devs.iter().zip(bins) { let mut ptr = b.as_ptr(); let bin_type; let spirv; unsafe { // 1. version let version = ptr.cast::().read(); ptr = ptr.add(size_of::()); match version { 1 => { // 2. size of the spirv let spirv_size = ptr.cast::().read(); ptr = ptr.add(size_of::()); // 3. binary_type bin_type = ptr.cast::().read(); ptr = ptr.add(size_of::()); // 4. the spirv assert!(b.as_ptr().add(BIN_HEADER_SIZE_V1) == ptr); assert!(b.len() == BIN_HEADER_SIZE_V1 + spirv_size as usize); spirv = Some(spirv::SPIRVBin::from_bin(slice::from_raw_parts( ptr, spirv_size as usize, ))); } _ => panic!("unknown version"), } } if let Some(spirv) = &spirv { for k in spirv.kernels() { kernels.insert(k); } } builds.insert( d.clone(), ProgramDevBuild { spirv: spirv, status: CL_BUILD_SUCCESS as cl_build_status, log: String::from(""), options: String::from(""), bin_type: bin_type, }, ); } let mut build = ProgramBuild { builds: builds, spec_constants: HashMap::new(), kernels: kernels.into_iter().collect(), kernel_builds: HashMap::new(), }; build.build_nirs(false); Arc::new(Self { base: CLObjectBase::new(), context: context, devs: devs, src: ProgramSourceType::Binary, kernel_count: AtomicU32::new(0), build: Mutex::new(build), }) } pub fn from_spirv(context: Arc, spirv: &[u8]) -> Arc { let builds = Self::create_default_builds(&context.devs); Arc::new(Self { base: CLObjectBase::new(), devs: context.devs.clone(), context: context, src: ProgramSourceType::Il(SPIRVBin::from_bin(spirv)), kernel_count: AtomicU32::new(0), build: Mutex::new(ProgramBuild { builds: builds, spec_constants: HashMap::new(), kernels: Vec::new(), kernel_builds: HashMap::new(), }), }) } fn build_info(&self) -> MutexGuard { self.build.lock().unwrap() } pub fn get_nir_kernel_build(&self, name: &str) -> NirKernelBuild { let info = self.build_info(); info.kernel_builds.get(name).unwrap().clone() } pub fn status(&self, dev: &Arc) -> cl_build_status { self.build_info().dev_build(dev).status } pub fn log(&self, dev: &Arc) -> String { self.build_info().dev_build(dev).log.clone() } pub fn bin_type(&self, dev: &Arc) -> cl_program_binary_type { self.build_info().dev_build(dev).bin_type } pub fn options(&self, dev: &Arc) -> String { self.build_info().dev_build(dev).options.clone() } // we need to precalculate the size pub fn bin_sizes(&self) -> Vec { let lock = self.build_info(); let mut res = Vec::new(); for d in &self.devs { let info = lock.dev_build(d); res.push( info.spirv .as_ref() .map_or(0, |s| s.to_bin().len() + BIN_HEADER_SIZE), ); } res } pub fn binaries(&self, vals: &[u8]) -> Vec<*mut u8> { // if the application didn't provide any pointers, just return the length of devices if vals.is_empty() { return vec![std::ptr::null_mut(); self.devs.len()]; } // vals is an array of pointers where we should write the device binaries into if vals.len() != self.devs.len() * size_of::<*const u8>() { panic!("wrong size") } let ptrs: &[*mut u8] = unsafe { slice::from_raw_parts(vals.as_ptr().cast(), vals.len() / size_of::<*mut u8>()) }; let lock = self.build_info(); for (i, d) in self.devs.iter().enumerate() { let mut ptr = ptrs[i]; let info = lock.dev_build(d); let spirv = info.spirv.as_ref().unwrap().to_bin(); unsafe { // 1. binary format version ptr.cast::().write(1); ptr = ptr.add(size_of::()); // 2. size of the spirv ptr.cast::().write(spirv.len() as u32); ptr = ptr.add(size_of::()); // 3. binary_type ptr.cast::().write(info.bin_type); ptr = ptr.add(size_of::()); // 4. the spirv assert!(ptrs[i].add(BIN_HEADER_SIZE) == ptr); ptr::copy_nonoverlapping(spirv.as_ptr(), ptr, spirv.len()); } } ptrs.to_vec() } pub fn kernel_signatures(&self, kernel_name: &str) -> HashSet> { let build = self.build_info(); let devs = build.devs_with_build(); if devs.is_empty() { return HashSet::new(); } devs.iter().map(|d| build.args(d, kernel_name)).collect() } pub fn kernels(&self) -> Vec { self.build_info().kernels.clone() } pub fn active_kernels(&self) -> bool { self.kernel_count.load(Ordering::Relaxed) != 0 } pub fn build(&self, dev: &Arc, options: String) -> bool { let lib = options.contains("-create-library"); let mut info = self.build_info(); if !self.do_compile(dev, options, &Vec::new(), &mut info) { return false; } let mut d = info.dev_build_mut(dev); let spirvs = [d.spirv.as_ref().unwrap()]; let (spirv, log) = spirv::SPIRVBin::link(&spirvs, lib); d.log.push_str(&log); d.spirv = spirv; if let Some(spirv) = &d.spirv { d.bin_type = if lib { CL_PROGRAM_BINARY_TYPE_LIBRARY } else { CL_PROGRAM_BINARY_TYPE_EXECUTABLE }; d.status = CL_BUILD_SUCCESS as cl_build_status; let mut kernels = spirv.kernels(); info.kernels.append(&mut kernels); info.build_nirs(self.is_src()); true } else { d.status = CL_BUILD_ERROR; d.bin_type = CL_PROGRAM_BINARY_TYPE_NONE; false } } fn do_compile( &self, dev: &Arc, options: String, headers: &[spirv::CLCHeader], info: &mut MutexGuard, ) -> bool { let mut d = info.dev_build_mut(dev); let (spirv, log) = match &self.src { ProgramSourceType::Il(spirv) => spirv.clone_on_validate(), ProgramSourceType::Src(src) => { let args = prepare_options(&options, dev); spirv::SPIRVBin::from_clc( src, &args, headers, get_disk_cache(), dev.cl_features(), &dev.spirv_extensions, dev.address_bits(), ) } // do nothing if we got a library or binary _ => { return true; } }; d.spirv = spirv; d.log = log; d.options = options; if d.spirv.is_some() { d.status = CL_BUILD_SUCCESS as cl_build_status; d.bin_type = CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT; true } else { d.status = CL_BUILD_ERROR; false } } pub fn compile( &self, dev: &Arc, options: String, headers: &[spirv::CLCHeader], ) -> bool { self.do_compile(dev, options, headers, &mut self.build_info()) } pub fn link( context: Arc, devs: &[Arc], progs: &[Arc], options: String, ) -> Arc { let devs: Vec> = devs.iter().map(|d| (*d).clone()).collect(); let mut builds = HashMap::new(); let mut kernels = HashSet::new(); let mut locks: Vec<_> = progs.iter().map(|p| p.build_info()).collect(); let lib = options.contains("-create-library"); for d in &devs { let bins: Vec<_> = locks .iter_mut() .map(|l| l.dev_build(d).spirv.as_ref().unwrap()) .collect(); let (spirv, log) = spirv::SPIRVBin::link(&bins, lib); let status; let bin_type; if let Some(spirv) = &spirv { for k in spirv.kernels() { kernels.insert(k); } status = CL_BUILD_SUCCESS as cl_build_status; bin_type = if lib { CL_PROGRAM_BINARY_TYPE_LIBRARY } else { CL_PROGRAM_BINARY_TYPE_EXECUTABLE }; } else { status = CL_BUILD_ERROR; bin_type = CL_PROGRAM_BINARY_TYPE_NONE; }; builds.insert( d.clone(), ProgramDevBuild { spirv: spirv, status: status, log: log, options: String::from(""), bin_type: bin_type, }, ); } let mut build = ProgramBuild { builds: builds, spec_constants: HashMap::new(), kernels: kernels.into_iter().collect(), kernel_builds: HashMap::new(), }; // Pre build nir kernels build.build_nirs(false); Arc::new(Self { base: CLObjectBase::new(), context: context, devs: devs, src: ProgramSourceType::Linked, kernel_count: AtomicU32::new(0), build: Mutex::new(build), }) } pub fn is_il(&self) -> bool { matches!(self.src, ProgramSourceType::Il(_)) } pub fn is_src(&self) -> bool { matches!(self.src, ProgramSourceType::Src(_)) } pub fn get_spec_constant_size(&self, spec_id: u32) -> u8 { match &self.src { ProgramSourceType::Il(il) => il .spec_constant(spec_id) .map_or(0, spirv::CLCSpecConstantType::size), _ => unreachable!(), } } pub fn set_spec_constant(&self, spec_id: u32, data: &[u8]) { let mut lock = self.build_info(); let mut val = nir_const_value::default(); match data.len() { 1 => val.u8_ = u8::from_ne_bytes(data.try_into().unwrap()), 2 => val.u16_ = u16::from_ne_bytes(data.try_into().unwrap()), 4 => val.u32_ = u32::from_ne_bytes(data.try_into().unwrap()), 8 => val.u64_ = u64::from_ne_bytes(data.try_into().unwrap()), _ => unreachable!("Spec constant with invalid size!"), }; lock.spec_constants.insert(spec_id, val); } }