From de46f070dd3ebee398b59778c017119a07e6c5e6 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 1 Oct 2026 20:01:23 +0800 Subject: [PATCH 1/2] cua_s1: load and validate native vision checkpoints on CPU --- recipe/cua_s1/native.md | 3 + recipe/cua_s1/native_vision.md | 64 + src/models/cua_s1/native/Cargo.toml | 6 +- .../cua_s1/native/examples/inspect_vision.rs | 22 + src/models/cua_s1/native/src/lib.rs | 2 + src/models/cua_s1/native/src/vision/mod.rs | 477 +++++ tests/cua_s1/fixtures/vision/README.md | 31 + .../fixtures/vision/adapter-tensors.json | 702 ++++++ .../fixtures/vision/adapter_config.json | 48 + .../cua_s1/fixtures/vision/base-tensors.json | 1887 +++++++++++++++++ tests/cua_s1/fixtures/vision/config.json | 104 + tests/cua_s1/vision_loader.rs | 466 ++++ 12 files changed, 3811 insertions(+), 1 deletion(-) create mode 100644 recipe/cua_s1/native_vision.md create mode 100644 src/models/cua_s1/native/examples/inspect_vision.rs create mode 100644 src/models/cua_s1/native/src/vision/mod.rs create mode 100644 tests/cua_s1/fixtures/vision/README.md create mode 100644 tests/cua_s1/fixtures/vision/adapter-tensors.json create mode 100644 tests/cua_s1/fixtures/vision/adapter_config.json create mode 100644 tests/cua_s1/fixtures/vision/base-tensors.json create mode 100644 tests/cua_s1/fixtures/vision/config.json create mode 100644 tests/cua_s1/vision_loader.rs diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index 018feff..ba27138 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -41,3 +41,6 @@ cargo test -p omni-cua-s1-native CUA_S1_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ cargo test --release -p omni-cua-s1-native --test kernels -- --ignored ``` + +For CPU-only structural inspection of the separate vision base weights and +multimodal adapter, see [native vision checkpoint inspection](native_vision.md). diff --git a/recipe/cua_s1/native_vision.md b/recipe/cua_s1/native_vision.md new file mode 100644 index 0000000..2a51b83 --- /dev/null +++ b/recipe/cua_s1/native_vision.md @@ -0,0 +1,64 @@ +# Cua-S1 native vision checkpoint inspection + +The model-owned Rust module `omni_cua_s1_native::vision` loads and structurally +validates the Qwen3.5-4B vision weights and Cua-S1 4B 0.2 **multimodal** LoRA on +CPU. It keeps the 297 BF16 base tensors and 100 FP32 adapter tensors separate. +The adapter contains 50 A/B pairs, rank 16, alpha 32, and scale 2. No weights are +merged or converted. This increment does not preprocess images, run the vision +encoder, call CUDA, or enable image requests in the native text worker. + +Prepare checkpoints from these pinned upstream revisions: + +| Checkpoint | Revision | Required files | +| --- | --- | --- | +| `Qwen/Qwen3.5-4B` | `851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a` | `config.json` and `model.safetensors`, or `model.safetensors.index.json` and its vision-bearing shards | +| `cua-ai/cua-s1-4b-0.2` multimodal adapter | `16818868b0cc7813808aae4e87b417657046ab79` | `adapter_config.json` and `adapter_model.safetensors` from the multimodal directory | + +Verify downloaded files against trusted upstream hashes before loading them. +Structural validation checks names, shapes, dtypes, configuration, and shard +mapping; it does **not** establish cryptographic identity or validate tensor +values. A bare base checkpoint or the `text` adapter is insufficient. + +From the repository root, pass exactly the base and multimodal adapter directories: + +```sh +cargo run --locked -p omni-cua-s1-native --example inspect_vision -- \ + weights/Qwen3.5-4B weights/cua-s1-4b-0.2/multimodal +``` + +The example reports the vision configuration, LoRA rank and scale, and tensor +counts. It needs neither a GPU nor a CUDA shared library. The index loader opens +only shards assigned visual tensors; unrelated language tensors in those shards +are ignored. Unexpected visual tensors, incompatible configuration, invalid +safetensors, missing tensors, mismatched index entries, and paths escaping the +checkpoint directory (including symlinks) are rejected. + +`VisionCheckpoint::load(base_dir, adapter_dir)` owns the memory maps and caches +validated tensor metadata. `config()` and `adapter()` expose the configuration; +`base_names()` / `adapter_names()` enumerate visual tensor names, and +`base_tensor(name)` / `adapter_tensor(name)` return borrowed safetensors +`TensorView`s with the original bytes. Views cannot outlive the checkpoint. +Accessors do not parse the file headers again. + +**Do not modify, replace in place, or truncate checkpoint files while a loaded +checkpoint or any of its views exists.** As with the existing native text +loader, callers must ensure that the memory-mapped files remain immutable. + +The supported layout is the pinned 4B configuration: 24 vision blocks, hidden +size 1024, intermediate size 4096, 16 heads, 2304 positions, output width 2560, +3 input channels, patch size 16, temporal patch size 2, spatial merge size 2, +`gelu_pytorch_tanh`, and no DeepStack. Text hidden size must equal vision output +width. LoRA targets both MLP matrices in every vision block and both merger +matrices. Math-changing PEFT options such as DoRA, rsLoRA, biases, target +exclusions, and custom rank/alpha patterns are unsupported. + +Run the CPU integration tests with: + +```sh +cargo test --locked -p omni-cua-s1-native --test vision_loader +``` + +Tests use sparse safetensors with real tensor shapes and small sentinel values; +no weight download or multi-gigabyte in-memory tensor allocation is required. +Their independent metadata oracle and provenance live in +[`tests/cua_s1/fixtures/vision/`](../../tests/cua_s1/fixtures/vision/). diff --git a/src/models/cua_s1/native/Cargo.toml b/src/models/cua_s1/native/Cargo.toml index 2e8caf6..e6903b7 100644 --- a/src/models/cua_s1/native/Cargo.toml +++ b/src/models/cua_s1/native/Cargo.toml @@ -17,8 +17,12 @@ half = "2.7.1" libloading = "0.8" memmap2 = "0.9.9" safetensors = "0.8.0" -serde = "1" +serde = { version = "1", features = ["derive"] } serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } # the onig regex backend, as in the Python tokenizers wheel tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "sync"] } + +[[test]] +name = "vision_loader" +path = "../../../../tests/cua_s1/vision_loader.rs" diff --git a/src/models/cua_s1/native/examples/inspect_vision.rs b/src/models/cua_s1/native/examples/inspect_vision.rs new file mode 100644 index 0000000..afffb19 --- /dev/null +++ b/src/models/cua_s1/native/examples/inspect_vision.rs @@ -0,0 +1,22 @@ +use anyhow::{Result, ensure}; +use omni_cua_s1_native::vision::VisionCheckpoint; + +fn main() -> Result<()> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + ensure!( + args.len() == 2, + "usage: inspect_vision BASE_DIR MULTIMODAL_ADAPTER_DIR" + ); + let checkpoint = VisionCheckpoint::load(&args[0], &args[1])?; + println!("Vision: {:#?}", checkpoint.config()); + println!("Base: {} BF16 tensors", checkpoint.base_names().count()); + println!( + "Adapter: {} FP32 tensors ({} pairs), rank {}, alpha {}, scale {}", + checkpoint.adapter_names().count(), + checkpoint.adapter_names().count() / 2, + checkpoint.adapter().rank, + checkpoint.adapter().alpha, + checkpoint.adapter().scale() + ); + Ok(()) +} diff --git a/src/models/cua_s1/native/src/lib.rs b/src/models/cua_s1/native/src/lib.rs index 0d3aa5f..547e704 100644 --- a/src/models/cua_s1/native/src/lib.rs +++ b/src/models/cua_s1/native/src/lib.rs @@ -7,3 +7,5 @@ pub mod cuda; pub mod engine; pub mod json; pub mod model; + +pub mod vision; diff --git a/src/models/cua_s1/native/src/vision/mod.rs b/src/models/cua_s1/native/src/vision/mod.rs new file mode 100644 index 0000000..16cc059 --- /dev/null +++ b/src/models/cua_s1/native/src/vision/mod.rs @@ -0,0 +1,477 @@ +//! CPU-only, structurally validated vision checkpoint storage. No LoRA merge or execution. +//! +//! Callers must keep checkpoint files immutable (including no truncation) for the +//! lifetime of the checkpoint and its borrowed views. Structural checks do not +//! verify upstream hashes or tensor values. + +use anyhow::{Context, Result, ensure}; +use memmap2::Mmap; +use safetensors::{ + Dtype, SafeTensors, + tensor::{TensorInfo, TensorView}, +}; +use serde::{ + Deserialize, + de::{self, MapAccess, Visitor}, +}; +use serde_json::Value; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, + fs::{self, File}, + path::{Component, Path, PathBuf}, +}; + +const BASE: &str = "model.visual."; +const ADAPTER: &str = "base_model.model.model.visual."; +type Inventory = BTreeMap>; + +/// The supported Qwen3.5-4B vision architecture; all fields are validated at load. +#[derive(Debug, Deserialize)] +pub struct VisionConfig { + pub depth: usize, + pub hidden_size: usize, + pub intermediate_size: usize, + pub num_heads: usize, + pub num_position_embeddings: usize, + pub out_hidden_size: usize, + pub in_channels: usize, + pub patch_size: usize, + pub temporal_patch_size: usize, + pub spatial_merge_size: usize, + pub hidden_act: String, + pub deepstack_visual_indexes: Vec, + pub model_type: String, +} +impl VisionConfig { + fn load(dir: &Path) -> Result { + let config: Value = serde_json::from_slice(&fs::read(inside(dir, "config.json")?)?)?; + let vision: Self = + serde_json::from_value(config["vision_config"].clone()).context("vision config")?; + let sizes = [ + vision.depth, + vision.hidden_size, + vision.intermediate_size, + vision.num_heads, + vision.num_position_embeddings, + vision.out_hidden_size, + vision.in_channels, + vision.patch_size, + vision.temporal_patch_size, + vision.spatial_merge_size, + ]; + ensure!( + sizes == [24, 1024, 4096, 16, 2304, 2560, 3, 16, 2, 2] + && vision.hidden_act == "gelu_pytorch_tanh" + && vision.deepstack_visual_indexes.is_empty() + && vision.model_type == "qwen3_5" + && config["model_type"] == "qwen3_5" + && config["text_config"]["hidden_size"].as_u64() + == Some(vision.out_hidden_size as u64), + "unsupported vision/text config: expected pinned Qwen3.5-4B layout" + ); + Ok(vision) + } + fn inventory(&self) -> Inventory { + let mut tensors = Inventory::new(); + let h = self.hidden_size; + let i = self.intermediate_size; + let mut linear = |name: String, output: usize, input: Option| { + tensors.insert( + format!("{BASE}{name}.weight"), + input.map_or_else(|| vec![output], |input| vec![output, input]), + ); + tensors.insert(format!("{BASE}{name}.bias"), vec![output]); + }; + for block in 0..self.depth { + for norm in ["norm1", "norm2"] { + linear(format!("blocks.{block}.{norm}"), h, None); + } + for (name, output, input) in [ + ("attn.qkv", 3 * h, h), + ("attn.proj", h, h), + ("mlp.linear_fc1", i, h), + ("mlp.linear_fc2", h, i), + ] { + linear(format!("blocks.{block}.{name}"), output, Some(input)); + } + } + let merged = h * self.spatial_merge_size * self.spatial_merge_size; + linear("merger.norm".into(), h, None); + linear("merger.linear_fc1".into(), merged, Some(merged)); + linear( + "merger.linear_fc2".into(), + self.out_hidden_size, + Some(merged), + ); + tensors.insert( + format!("{BASE}patch_embed.proj.weight"), + vec![ + h, + self.in_channels, + self.temporal_patch_size, + self.patch_size, + self.patch_size, + ], + ); + tensors.insert(format!("{BASE}patch_embed.proj.bias"), vec![h]); + tensors.insert( + format!("{BASE}pos_embed.weight"), + vec![self.num_position_embeddings, h], + ); + tensors + } +} + +/// Inference LoRA parameters; base and adapter bytes remain separate. +#[derive(Debug)] +pub struct AdapterConfig { + pub rank: usize, + pub alpha: usize, +} +impl AdapterConfig { + pub fn scale(&self) -> f64 { + self.alpha as f64 / self.rank as f64 + } + fn load(dir: &Path) -> Result { + let config: Value = + serde_json::from_slice(&fs::read(inside(dir, "adapter_config.json")?)?)?; + let c = config + .as_object() + .context("adapter config must be an object")?; + for (key, value) in c { + let supported = match key.as_str() { + "r" => value == 16, + "lora_alpha" => value == 32, + "peft_type" => value == "LORA", + "bias" => value == "none", + "base_model_name_or_path" => value == "Qwen/Qwen3.5-4B", + "task_type" => value == "CAUSAL_LM", + "lora_bias" + | "use_dora" + | "use_rslora" + | "use_qalora" + | "fan_in_fan_out" + | "ensure_weight_tying" => value == false, + "rank_pattern" | "alpha_pattern" | "loftq_config" => { + value.as_object().is_some_and(|v| v.is_empty()) + } + "exclude_modules" + | "modules_to_save" + | "layers_to_transform" + | "layers_pattern" + | "layer_replication" + | "target_parameters" + | "trainable_token_indices" + | "alora_invocation_tokens" + | "arrow_config" + | "corda_config" + | "eva_config" + | "megatron_config" => value.is_null(), + // Training/serialization metadata does not change ordinary inference LoRA. + "auto_mapping" | "inference_mode" | "init_lora_weights" | "lora_dropout" + | "megatron_core" | "peft_version" | "qalora_group_size" | "revision" + | "target_modules" => true, + _ => false, + }; + ensure!( + supported, + "unsupported adapter config option {key}: {value}" + ); + } + for (key, value) in [ + ("r", Value::from(16)), + ("lora_alpha", Value::from(32)), + ("peft_type", Value::from("LORA")), + ("bias", Value::from("none")), + ] { + ensure!(config[key] == value, "unsupported adapter config {key}"); + } + let targets = config["target_modules"] + .as_array() + .context("adapter config target_modules must be an array")?; + let expected = BTreeSet::from([ + "up_proj", + "k_proj", + "linear_fc1", + "q_proj", + "linear_fc2", + "down_proj", + "gate_proj", + "o_proj", + "v_proj", + ]); + let actual: BTreeSet<_> = targets.iter().filter_map(Value::as_str).collect(); + ensure!( + actual == expected && targets.len() == expected.len(), + "adapter config requires the full multimodal target_modules" + ); + Ok(Self { + rank: 16, + alpha: 32, + }) + } + fn inventory(&self, base: &Inventory) -> Inventory { + let mut tensors = Inventory::new(); + for (name, shape) in base { + if name.ends_with(".weight") + && (name.contains(".linear_fc1.") || name.contains(".linear_fc2.")) + { + let module = name + .strip_prefix(BASE) + .unwrap() + .strip_suffix(".weight") + .unwrap(); + tensors.insert( + format!("{ADAPTER}{module}.lora_A.weight"), + vec![self.rank, shape[1]], + ); + tensors.insert( + format!("{ADAPTER}{module}.lora_B.weight"), + vec![shape[0], self.rank], + ); + } + } + tensors + } +} + +/// Immutable mmap storage for one base checkpoint and its multimodal adapter. +pub struct VisionCheckpoint { + config: VisionConfig, + adapter: AdapterConfig, + base: TensorStore, + lora: TensorStore, +} +impl VisionCheckpoint { + /// Loads and validates headers on CPU. Keep the files immutable while mapped. + pub fn load(base_dir: impl AsRef, adapter_dir: impl AsRef) -> Result { + let base_dir = fs::canonicalize(base_dir).context("base checkpoint directory")?; + let adapter_dir = fs::canonicalize(adapter_dir).context("adapter checkpoint directory")?; + let config = VisionConfig::load(&base_dir).context("base config")?; + let adapter = AdapterConfig::load(&adapter_dir).context("adapter config")?; + let expected = config.inventory(); + let lora_expected = adapter.inventory(&expected); + let index_path = base_dir.join("model.safetensors.index.json"); + let index = if index_path.try_exists()? { + #[derive(Deserialize)] + struct Index { + weight_map: UniqueMap, + } + let index: Index = serde_json::from_slice(&fs::read(inside( + &base_dir, + "model.safetensors.index.json", + )?)?) + .context("safetensors index")?; + let map = index.weight_map.0; + for name in map.keys().filter(|n| is_visual(n)) { + ensure!( + expected.contains_key(name), + "unexpected visual tensor in index: {name}" + ); + } + for name in expected.keys() { + ensure!( + map.contains_key(name), + "missing visual tensor in index: {name}" + ); + } + Some(map) + } else { + None + }; + let files: BTreeSet = match &index { + Some(index) => expected.keys().map(|n| index[n].clone()).collect(), + None => BTreeSet::from(["model.safetensors".into()]), + }; + let base = TensorStore::load(&base_dir, files, &expected, Dtype::BF16, index.as_ref())?; + let lora = TensorStore::load( + &adapter_dir, + BTreeSet::from(["adapter_model.safetensors".into()]), + &lora_expected, + Dtype::F32, + None, + )?; + Ok(Self { + config, + adapter, + base, + lora, + }) + } + pub fn config(&self) -> &VisionConfig { + &self.config + } + pub fn adapter(&self) -> &AdapterConfig { + &self.adapter + } + pub fn base_names(&self) -> impl Iterator { + self.base.tensors.keys().map(String::as_str) + } + pub fn adapter_names(&self) -> impl Iterator { + self.lora.tensors.keys().map(String::as_str) + } + pub fn base_tensor(&self, name: &str) -> Result> { + self.base.tensor(name) + } + pub fn adapter_tensor(&self, name: &str) -> Result> { + self.lora.tensor(name) + } +} + +struct TensorStore { + maps: Vec, + tensors: BTreeMap, +} +impl TensorStore { + fn load( + dir: &Path, + files: BTreeSet, + expected: &Inventory, + dtype: Dtype, + index: Option<&BTreeMap>, + ) -> Result { + let mut store = Self { + maps: Vec::new(), + tensors: BTreeMap::new(), + }; + for filename in files { + let path = inside(dir, &filename)?; + let file = File::open(&path).with_context(|| format!("open {}", path.display()))?; + // SAFETY: callers must not mutate or truncate checkpoint files while mapped. + let map = unsafe { Mmap::map(&file) } + .with_context(|| format!("mmap safetensors {}", path.display()))?; + validate_header(&map) + .with_context(|| format!("safetensors header {}", path.display()))?; + let (header_len, metadata) = SafeTensors::read_metadata(&map) + .with_context(|| format!("safetensors {}", path.display()))?; + for (name, info) in metadata.tensors() { + if !is_visual(&name) { + continue; + } + let shape = expected + .get(&name) + .with_context(|| format!("unexpected visual tensor {name} in {filename}"))?; + if let Some(index) = index { + ensure!( + index.get(&name) == Some(&filename), + "index mismatch for {name} in {filename}" + ); + } + ensure!( + &info.shape == shape, + "shape mismatch for {name}: {:?}, expected {shape:?}", + info.shape + ); + ensure!( + info.dtype == dtype, + "dtype mismatch for {name}: {:?}, expected {dtype:?}", + info.dtype + ); + let mut info = info.clone(); + info.data_offsets.0 += 8 + header_len; + info.data_offsets.1 += 8 + header_len; + ensure!( + store + .tensors + .insert(name.clone(), (store.maps.len(), info)) + .is_none(), + "duplicate visual tensor {name}" + ); + } + store.maps.push(map); + } + for name in expected.keys() { + ensure!( + store.tensors.contains_key(name), + "missing visual tensor {name}" + ); + } + Ok(store) + } + fn tensor(&self, name: &str) -> Result> { + let (shard, info) = self + .tensors + .get(name) + .with_context(|| format!("unknown visual tensor {name}"))?; + Ok(TensorView::new( + info.dtype, + info.shape.clone(), + &self.maps[*shard][info.data_offsets.0..info.data_offsets.1], + )?) + } +} +// Bound all offsets before safetensors 0.8 adds payload size to header size: +// its final length check uses unchecked addition, even for ignored language tensors. +fn validate_header(bytes: &[u8]) -> Result<()> { + let length_bytes = bytes.get(..8).context("missing header length")?; + let header_len = usize::try_from(u64::from_le_bytes(length_bytes.try_into()?))?; + // Match safetensors 0.8's header allocation limit. + ensure!(header_len <= 100_000_000, "header too large"); + let data_start = header_len + .checked_add(8) + .context("header length overflow")?; + let header = bytes.get(8..data_start).context("truncated header")?; + let payload_len = bytes.len() - data_start; + let entries: UniqueMap = serde_json::from_slice(header)?; + for (name, entry) in entries.0 { + if name == "__metadata__" { + continue; + } + let (start, end): (usize, usize) = serde_json::from_value(entry["data_offsets"].clone()) + .with_context(|| format!("invalid offsets for {name}"))?; + ensure!( + start <= end && end <= payload_len, + "tensor offsets exceed payload for {name}" + ); + } + Ok(()) +} +fn is_visual(name: &str) -> bool { + name.split('.').any(|part| part == "visual") +} +fn inside(dir: &Path, name: &str) -> Result { + ensure!( + !name.is_empty() + && Path::new(name) + .components() + .all(|c| matches!(c, Component::Normal(_))), + "path must remain inside checkpoint directory: {name}" + ); + let resolved = + fs::canonicalize(dir.join(name)).with_context(|| format!("checkpoint file {name}"))?; + ensure!( + resolved.starts_with(dir), + "path escapes checkpoint directory: {name}" + ); + Ok(resolved) +} + +// serde_json's ordinary maps overwrite repeated keys. Reject ambiguous headers/indexes. +struct UniqueMap(BTreeMap); +impl<'de, T: Deserialize<'de>> Deserialize<'de> for UniqueMap { + fn deserialize>( + deserializer: D, + ) -> std::result::Result { + struct UniqueVisitor(std::marker::PhantomData); + impl<'de, T: Deserialize<'de>> Visitor<'de> for UniqueVisitor { + type Value = UniqueMap; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("an object with unique names") + } + fn visit_map>( + self, + mut map: M, + ) -> std::result::Result { + let mut entries = BTreeMap::new(); + while let Some((key, value)) = map.next_entry::()? { + if entries.insert(key.clone(), value).is_some() { + return Err(de::Error::custom(format!("duplicate JSON key: {key}"))); + } + } + Ok(UniqueMap(entries)) + } + } + deserializer.deserialize_map(UniqueVisitor(std::marker::PhantomData)) + } +} diff --git a/tests/cua_s1/fixtures/vision/README.md b/tests/cua_s1/fixtures/vision/README.md new file mode 100644 index 0000000..21a8ca6 --- /dev/null +++ b/tests/cua_s1/fixtures/vision/README.md @@ -0,0 +1,31 @@ +# Pinned vision metadata fixtures + +These fixtures are an independent test oracle captured from upstream metadata, +not generated by the production Rust inventory builder: + +- Base: `Qwen/Qwen3.5-4B`, revision + `851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a`. +- Adapter: `cua-ai/cua-s1-4b-0.2`, revision + `16818868b0cc7813808aae4e87b417657046ab79`, multimodal adapter. +- `config.json` and `adapter_config.json`: upstream configuration files with LF line endings. +- `base-tensors.json`: visual names, shapes, and dtypes read from the pinned + base safetensors headers (prefix `model.visual.`): 297 BF16 tensors. +- `adapter-tensors.json`: visual names, shapes, and dtypes read from the pinned + multimodal adapter safetensors header (prefix + `base_model.model.model.visual.`): 100 FP32 tensors / 50 LoRA pairs. + +No weight payloads are included. Tests construct sparse files with these shapes +and sentinel values. Production code never reads these fixtures. Fixture +matching proves structural compatibility, not checkpoint identity: verify real +checkpoint downloads against trusted upstream hashes separately. + +Exact pinned sources: + +- [Base configuration](https://huggingface.co/Qwen/Qwen3.5-4B/resolve/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/config.json) +- [Multimodal adapter configuration](https://huggingface.co/cua-ai/cua-s1-4b-0.2/resolve/16818868b0cc7813808aae4e87b417657046ab79/multimodal/adapter_config.json) +- [Base second shard, containing the vision tensors](https://huggingface.co/Qwen/Qwen3.5-4B/resolve/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/model.safetensors-00002-of-00002.safetensors) +- [Multimodal adapter safetensors](https://huggingface.co/cua-ai/cua-s1-4b-0.2/resolve/16818868b0cc7813808aae4e87b417657046ab79/multimodal/adapter_model.safetensors) + +The tensor inventories retain only the `shape` and `dtype` fields of the visual +entries in those safetensors headers; byte offsets and language tensors are not +part of the oracle. diff --git a/tests/cua_s1/fixtures/vision/adapter-tensors.json b/tests/cua_s1/fixtures/vision/adapter-tensors.json new file mode 100644 index 0000000..ac90604 --- /dev/null +++ b/tests/cua_s1/fixtures/vision/adapter-tensors.json @@ -0,0 +1,702 @@ +{ + "base_model.model.model.visual.blocks.0.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.0.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.0.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.0.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.1.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.1.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.1.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.1.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.10.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.10.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.10.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.10.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.11.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.11.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.11.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.11.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.12.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.12.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.12.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.12.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.13.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.13.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.13.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.13.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.14.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.14.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.14.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.14.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.15.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.15.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.15.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.15.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.16.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.16.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.16.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.16.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.17.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.17.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.17.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.17.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.18.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.18.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.18.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.18.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.19.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.19.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.19.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.19.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.2.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.2.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.2.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.2.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.20.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.20.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.20.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.20.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.21.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.21.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.21.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.21.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.22.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.22.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.22.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.22.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.23.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.23.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.23.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.23.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.3.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.3.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.3.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.3.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.4.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.4.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.4.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.4.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.5.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.5.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.5.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.5.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.6.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.6.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.6.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.6.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.7.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.7.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.7.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.7.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.8.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.8.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.8.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.8.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.blocks.9.mlp.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 1024 + ] + }, + "base_model.model.model.visual.blocks.9.mlp.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.blocks.9.mlp.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.blocks.9.mlp.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 1024, + 16 + ] + }, + "base_model.model.model.visual.merger.linear_fc1.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.merger.linear_fc1.lora_B.weight": { + "dtype": "F32", + "shape": [ + 4096, + 16 + ] + }, + "base_model.model.model.visual.merger.linear_fc2.lora_A.weight": { + "dtype": "F32", + "shape": [ + 16, + 4096 + ] + }, + "base_model.model.model.visual.merger.linear_fc2.lora_B.weight": { + "dtype": "F32", + "shape": [ + 2560, + 16 + ] + } +} diff --git a/tests/cua_s1/fixtures/vision/adapter_config.json b/tests/cua_s1/fixtures/vision/adapter_config.json new file mode 100644 index 0000000..52b492f --- /dev/null +++ b/tests/cua_s1/fixtures/vision/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "k_proj", + "linear_fc1", + "q_proj", + "linear_fc2", + "down_proj", + "gate_proj", + "o_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} diff --git a/tests/cua_s1/fixtures/vision/base-tensors.json b/tests/cua_s1/fixtures/vision/base-tensors.json new file mode 100644 index 0000000..0c2d0b9 --- /dev/null +++ b/tests/cua_s1/fixtures/vision/base-tensors.json @@ -0,0 +1,1887 @@ +{ + "model.visual.blocks.0.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.0.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.0.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.0.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.0.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.0.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.0.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.0.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.0.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.0.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.0.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.0.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.1.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.1.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.1.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.1.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.1.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.1.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.1.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.1.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.1.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.1.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.1.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.1.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.10.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.10.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.10.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.10.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.10.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.10.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.10.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.10.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.10.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.10.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.10.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.10.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.11.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.11.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.11.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.11.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.11.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.11.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.11.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.11.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.11.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.11.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.11.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.11.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.12.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.12.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.12.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.12.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.12.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.12.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.12.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.12.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.12.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.12.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.12.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.12.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.13.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.13.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.13.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.13.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.13.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.13.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.13.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.13.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.13.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.13.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.13.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.13.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.14.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.14.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.14.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.14.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.14.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.14.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.14.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.14.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.14.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.14.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.14.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.14.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.15.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.15.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.15.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.15.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.15.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.15.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.15.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.15.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.15.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.15.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.15.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.15.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.16.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.16.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.16.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.16.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.16.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.16.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.16.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.16.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.16.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.16.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.16.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.16.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.17.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.17.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.17.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.17.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.17.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.17.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.17.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.17.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.17.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.17.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.17.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.17.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.18.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.18.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.18.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.18.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.18.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.18.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.18.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.18.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.18.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.18.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.18.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.18.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.19.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.19.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.19.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.19.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.19.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.19.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.19.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.19.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.19.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.19.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.19.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.19.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.2.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.2.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.2.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.2.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.2.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.2.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.2.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.2.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.2.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.2.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.2.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.2.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.20.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.20.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.20.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.20.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.20.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.20.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.20.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.20.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.20.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.20.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.20.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.20.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.21.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.21.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.21.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.21.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.21.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.21.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.21.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.21.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.21.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.21.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.21.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.21.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.22.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.22.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.22.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.22.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.22.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.22.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.22.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.22.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.22.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.22.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.22.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.22.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.23.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.23.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.23.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.23.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.23.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.23.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.23.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.23.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.23.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.23.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.23.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.23.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.3.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.3.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.3.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.3.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.3.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.3.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.3.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.3.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.3.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.3.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.3.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.3.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.4.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.4.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.4.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.4.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.4.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.4.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.4.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.4.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.4.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.4.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.4.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.4.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.5.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.5.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.5.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.5.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.5.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.5.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.5.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.5.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.5.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.5.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.5.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.5.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.6.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.6.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.6.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.6.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.6.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.6.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.6.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.6.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.6.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.6.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.6.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.6.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.7.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.7.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.7.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.7.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.7.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.7.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.7.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.7.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.7.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.7.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.7.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.7.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.8.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.8.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.8.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.8.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.8.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.8.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.8.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.8.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.8.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.8.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.8.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.8.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.9.attn.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.9.attn.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 1024 + ] + }, + "model.visual.blocks.9.attn.qkv.bias": { + "dtype": "BF16", + "shape": [ + 3072 + ] + }, + "model.visual.blocks.9.attn.qkv.weight": { + "dtype": "BF16", + "shape": [ + 3072, + 1024 + ] + }, + "model.visual.blocks.9.mlp.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.blocks.9.mlp.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 1024 + ] + }, + "model.visual.blocks.9.mlp.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.9.mlp.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 4096 + ] + }, + "model.visual.blocks.9.norm1.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.9.norm1.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.9.norm2.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.blocks.9.norm2.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.merger.linear_fc1.bias": { + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "model.visual.merger.linear_fc1.weight": { + "dtype": "BF16", + "shape": [ + 4096, + 4096 + ] + }, + "model.visual.merger.linear_fc2.bias": { + "dtype": "BF16", + "shape": [ + 2560 + ] + }, + "model.visual.merger.linear_fc2.weight": { + "dtype": "BF16", + "shape": [ + 2560, + 4096 + ] + }, + "model.visual.merger.norm.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.merger.norm.weight": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.patch_embed.proj.bias": { + "dtype": "BF16", + "shape": [ + 1024 + ] + }, + "model.visual.patch_embed.proj.weight": { + "dtype": "BF16", + "shape": [ + 1024, + 3, + 2, + 16, + 16 + ] + }, + "model.visual.pos_embed.weight": { + "dtype": "BF16", + "shape": [ + 2304, + 1024 + ] + } +} diff --git a/tests/cua_s1/fixtures/vision/config.json b/tests/cua_s1/fixtures/vision/config.json new file mode 100644 index 0000000..cca4041 --- /dev/null +++ b/tests/cua_s1/fixtures/vision/config.json @@ -0,0 +1,104 @@ +{ + "architectures": [ + "Qwen3_5ForConditionalGeneration" + ], + "image_token_id": 248056, + "model_type": "qwen3_5", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_hidden_layers": 32, + "num_key_value_heads": 4, + "rms_norm_eps": 1e-06, + "tie_word_embeddings": true, + "use_cache": true, + "vocab_size": 248320, + "mamba_ssm_dtype": "float32", + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "rope_type": "default", + "rope_theta": 10000000, + "partial_rotary_factor": 0.25 + } + }, + "tie_word_embeddings": true, + "transformers_version": "4.57.0.dev0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 24, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4096, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/tests/cua_s1/vision_loader.rs b/tests/cua_s1/vision_loader.rs new file mode 100644 index 0000000..8de73e2 --- /dev/null +++ b/tests/cua_s1/vision_loader.rs @@ -0,0 +1,466 @@ +use omni_cua_s1_native::vision::VisionCheckpoint; +use safetensors::Dtype; +use serde_json::{Value, json}; +use std::{ + collections::BTreeMap, + fs::{self, File}, + io::{Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + sync::atomic::{AtomicUsize, Ordering}, +}; + +const BASE: &str = "model.visual."; +const ADAPTER: &str = "base_model.model.model.visual."; +type Inventory = BTreeMap; +fn inventory(adapter: bool) -> Inventory { + serde_json::from_str(if adapter { + include_str!("fixtures/vision/adapter-tensors.json") + } else { + include_str!("fixtures/vision/base-tensors.json") + }) + .unwrap() +} +fn sparse(path: &Path, tensors: &Inventory) { + let mut offset = 0u64; + let mut header = serde_json::Map::new(); + for (name, tensor) in tensors { + let bytes = tensor["shape"] + .as_array() + .unwrap() + .iter() + .map(|n| n.as_u64().unwrap()) + .product::() + * if tensor["dtype"] == "BF16" { 2 } else { 4 }; + header.insert(name.clone(), json!({"dtype":tensor["dtype"],"shape":tensor["shape"],"data_offsets":[offset, offset+bytes]})); + offset += bytes; + } + let header = serde_json::to_vec(&header).unwrap(); + let mut file = File::create(path).unwrap(); + file.write_all(&(header.len() as u64).to_le_bytes()) + .unwrap(); + file.write_all(&header).unwrap(); + file.set_len(8 + header.len() as u64 + offset).unwrap(); + for tensor in serde_json::from_slice::(&header) + .unwrap() + .as_object() + .unwrap() + .values() + { + file.seek(SeekFrom::Start( + 8 + header.len() as u64 + tensor["data_offsets"][0].as_u64().unwrap(), + )) + .unwrap(); + file.write_all(if tensor["dtype"] == "BF16" { + &[0x80, 0x3f] + } else { + &[0, 0, 0x80, 0x3f] + }) + .unwrap(); + } +} +struct Fixture { + root: PathBuf, + base: PathBuf, + adapter: PathBuf, +} +impl Fixture { + fn new() -> Self { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let root = std::env::temp_dir().join(format!( + "cua-vision-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let base = root.join("base"); + let adapter = root.join("adapter"); + fs::create_dir_all(&base).unwrap(); + fs::create_dir_all(&adapter).unwrap(); + fs::write( + base.join("config.json"), + include_str!("fixtures/vision/config.json"), + ) + .unwrap(); + fs::write( + adapter.join("adapter_config.json"), + include_str!("fixtures/vision/adapter_config.json"), + ) + .unwrap(); + sparse(&base.join("model.safetensors"), &inventory(false)); + sparse(&adapter.join("adapter_model.safetensors"), &inventory(true)); + Self { + root, + base, + adapter, + } + } + fn load(&self) -> anyhow::Result { + VisionCheckpoint::load(&self.base, &self.adapter) + } + fn reject(&self, expected: &str) { + let error = match self.load() { + Ok(_) => panic!("accepted invalid checkpoint: {expected}"), + Err(error) => format!("{error:#}"), + }; + assert!( + error.contains(expected), + "expected {expected:?}, got {error}" + ); + } + fn config(&self, adapter: bool, key: &str, value: Value) { + let path = if adapter { + self.adapter.join("adapter_config.json") + } else { + self.base.join("config.json") + }; + let mut config: Value = serde_json::from_str(if adapter { + include_str!("fixtures/vision/adapter_config.json") + } else { + include_str!("fixtures/vision/config.json") + }) + .unwrap(); + if adapter { + config[key] = value; + } else if key == "text_hidden" { + config["text_config"]["hidden_size"] = value; + } else { + config["vision_config"][key] = value; + } + fs::write(path, serde_json::to_vec(&config).unwrap()).unwrap(); + } + fn shards(&self) -> BTreeMap { + let (a, b): (Inventory, Inventory) = inventory(false) + .into_iter() + .partition(|(n, _)| n.contains("blocks.")); + sparse(&self.base.join("a.safetensors"), &a); + sparse(&self.base.join("b.safetensors"), &b); + let mut map: BTreeMap = a + .keys() + .map(|n| (n.clone(), "a.safetensors".into())) + .chain(b.keys().map(|n| (n.clone(), "b.safetensors".into()))) + .collect(); + map.insert( + "model.language_model.weight".into(), + "absent-language.safetensors".into(), + ); + self.index(&map); + fs::remove_file(self.base.join("model.safetensors")).unwrap(); + map + } + fn index(&self, map: &BTreeMap) { + fs::write( + self.base.join("model.safetensors.index.json"), + serde_json::to_vec(&json!({"weight_map":map})).unwrap(), + ) + .unwrap(); + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } +} +fn check(checkpoint: &VisionCheckpoint) { + assert_eq!(checkpoint.config().depth, 24); + assert_eq!(checkpoint.config().hidden_size, 1024); + assert_eq!(checkpoint.adapter().rank, 16); + assert_eq!(checkpoint.adapter().alpha, 32); + assert_eq!(checkpoint.adapter().scale(), 2.0); + for adapter in [false, true] { + let expected = inventory(adapter); + let names: Vec<_> = if adapter { + checkpoint.adapter_names().collect() + } else { + checkpoint.base_names().collect() + }; + assert_eq!(names.len(), if adapter { 100 } else { 297 }); + for (name, spec) in expected { + let tensor = if adapter { + checkpoint.adapter_tensor(&name) + } else { + checkpoint.base_tensor(&name) + } + .unwrap(); + assert_eq!( + tensor.shape(), + serde_json::from_value::>(spec["shape"].clone()).unwrap() + ); + assert_eq!( + tensor.dtype(), + if adapter { Dtype::F32 } else { Dtype::BF16 } + ); + assert_eq!( + &tensor.data()[..if adapter { 4 } else { 2 }], + if adapter { + &[0, 0, 0x80, 0x3f][..] + } else { + &[0x80, 0x3f][..] + } + ); + } + } + assert!(checkpoint.base_tensor("missing").is_err()); +} +#[test] +fn valid_single_matches_independent_inventory() { + let f = Fixture::new(); + check(&f.load().unwrap()); +} +#[test] +fn valid_shards_skip_language_only_files() { + let f = Fixture::new(); + f.shards(); + check(&f.load().unwrap()); +} +#[test] +fn ignores_language_tensors() { + let f = Fixture::new(); + for adapter in [false, true] { + let mut inv = inventory(adapter); + inv.insert( + "unrelated.language.weight".into(), + json!({"shape":[1],"dtype":"F32"}), + ); + sparse( + &if adapter { + f.adapter.join("adapter_model.safetensors") + } else { + f.base.join("model.safetensors") + }, + &inv, + ); + } + check(&f.load().unwrap()); +} +#[test] +fn rejects_missing_wrong_and_extra_tensors() { + let f = Fixture::new(); + for adapter in [false, true] { + let path = if adapter { + f.adapter.join("adapter_model.safetensors") + } else { + f.base.join("model.safetensors") + }; + let original = inventory(adapter); + let name = original.keys().next().unwrap().clone(); + let mut inv = original.clone(); + inv.remove(&name); + sparse(&path, &inv); + f.reject("missing"); + let mut inv = original.clone(); + inv.get_mut(&name).unwrap()["shape"] = json!([1]); + sparse(&path, &inv); + f.reject("shape"); + let mut inv = original.clone(); + inv.get_mut(&name).unwrap()["dtype"] = json!(if adapter { "BF16" } else { "F32" }); + sparse(&path, &inv); + f.reject("dtype"); + let mut inv = original.clone(); + inv.insert( + format!("{}unexpected.weight", if adapter { ADAPTER } else { BASE }), + json!({"shape":[1],"dtype":"F32"}), + ); + sparse(&path, &inv); + f.reject("unexpected"); + sparse(&path, &original); + } +} +#[test] +fn rejects_truncated_and_malformed_safetensors() { + let f = Fixture::new(); + for adapter in [false, true] { + let path = if adapter { + f.adapter.join("adapter_model.safetensors") + } else { + f.base.join("model.safetensors") + }; + let len = fs::metadata(&path).unwrap().len(); + File::options() + .write(true) + .open(&path) + .unwrap() + .set_len(len - 1) + .unwrap(); + f.reject("safetensors"); + fs::write(&path, b"invalid").unwrap(); + f.reject("safetensors"); + sparse(&path, &inventory(adapter)); + } +} +#[test] +fn rejects_index_missing_extra_misrouting_and_duplicates() { + let f = Fixture::new(); + let original = f.shards(); + let name = inventory(false).keys().next().unwrap().clone(); + let mut map = original.clone(); + map.remove(&name); + f.index(&map); + f.reject("missing"); + let mut map = original.clone(); + map.insert(format!("{BASE}unexpected"), "a.safetensors".into()); + f.index(&map); + f.reject("unexpected"); + let mut map = original.clone(); + map.insert(name.clone(), "b.safetensors".into()); + f.index(&map); + f.reject("index"); + f.index(&original); + let mut b: Inventory = inventory(false) + .into_iter() + .filter(|(n, _)| !n.contains("blocks.")) + .collect(); + b.insert(name.clone(), inventory(false)[&name].clone()); + sparse(&f.base.join("b.safetensors"), &b); + f.reject("index"); + fs::write( + f.base.join("model.safetensors.index.json"), + format!("{{\"weight_map\":{{\"{name}\":\"a.safetensors\",\"{name}\":\"b.safetensors\"}}}}"), + ) + .unwrap(); + f.reject("duplicate"); +} +#[test] +fn rejects_index_paths_outside_directory() { + let f = Fixture::new(); + let original = f.shards(); + let name = inventory(false).keys().next().unwrap().clone(); + for path in ["../outside.safetensors", "/tmp/outside.safetensors"] { + let mut map = original.clone(); + map.insert(name.clone(), path.into()); + f.index(&map); + f.reject("checkpoint directory"); + } +} +#[cfg(unix)] +#[test] +fn rejects_symlink_escape() { + use std::os::unix::fs::symlink; + let f = Fixture::new(); + fs::rename( + f.base.join("model.safetensors"), + f.root.join("outside.safetensors"), + ) + .unwrap(); + symlink( + f.root.join("outside.safetensors"), + f.base.join("model.safetensors"), + ) + .unwrap(); + f.reject("checkpoint directory"); +} +#[test] +fn validates_full_vision_configuration() { + let f = Fixture::new(); + for (key, value) in [ + ("depth", json!(23)), + ("hidden_size", json!(512)), + ("intermediate_size", json!(2048)), + ("num_heads", json!(8)), + ("num_position_embeddings", json!(1024)), + ("out_hidden_size", json!(2048)), + ("in_channels", json!(1)), + ("patch_size", json!(14)), + ("temporal_patch_size", json!(1)), + ("spatial_merge_size", json!(1)), + ("hidden_act", json!("gelu")), + ("deepstack_visual_indexes", json!([1])), + ("text_hidden", json!(2048)), + ] { + f.config(false, key, value); + f.reject("config"); + } +} +#[test] +fn rejects_adapter_math_and_target_changes() { + let f = Fixture::new(); + for (key, value) in [ + ("r", json!(8)), + ("lora_alpha", json!(16)), + ("bias", json!("all")), + ("lora_bias", json!(true)), + ("use_dora", json!(true)), + ("use_rslora", json!(true)), + ("use_qalora", json!(true)), + ("fan_in_fan_out", json!(true)), + ("rank_pattern", json!({"linear_fc1":8})), + ("alpha_pattern", json!({"linear_fc1":16})), + ("exclude_modules", json!(["linear_fc1"])), + ("target_modules", json!(["up_proj", "down_proj"])), + ("modules_to_save", json!(["merger"])), + ("layers_to_transform", json!([1])), + ("target_parameters", json!(["weight"])), + ("layer_replication", json!([[0, 1]])), + ("alora_invocation_tokens", json!([1])), + ("peft_type", json!("IA3")), + ] { + f.config(true, key, value); + f.reject("adapter config"); + } +} +#[test] +fn requires_multimodal_adapter() { + let f = Fixture::new(); + fs::remove_file(f.adapter.join("adapter_model.safetensors")).unwrap(); + f.reject("adapter_model.safetensors"); + sparse( + &f.adapter.join("adapter_model.safetensors"), + &BTreeMap::from([( + "base_model.model.model.language_model.weight".into(), + json!({"shape":[1],"dtype":"F32"}), + )]), + ); + f.reject("missing"); +} + +#[test] +fn rejects_duplicate_visual_keys_in_safetensors_header() { + use std::io::Read; + let f = Fixture::new(); + let path = f.base.join("model.safetensors"); + let mut file = File::open(&path).unwrap(); + let total = file.metadata().unwrap().len(); + let mut size = [0; 8]; + file.read_exact(&mut size).unwrap(); + let old_len = u64::from_le_bytes(size); + let mut header = vec![0; old_len as usize]; + file.read_exact(&mut header).unwrap(); + drop(file); + let entries: BTreeMap = serde_json::from_slice(&header).unwrap(); + let (name, info) = entries.first_key_value().unwrap(); + let mut duplicate = String::from_utf8(header).unwrap(); + duplicate.pop(); + duplicate.push_str(&format!( + ",{}:{info}}}", + serde_json::to_string(name).unwrap() + )); + let mut file = File::create(&path).unwrap(); + file.write_all(&(duplicate.len() as u64).to_le_bytes()) + .unwrap(); + file.write_all(duplicate.as_bytes()).unwrap(); + file.set_len(total - old_len + duplicate.len() as u64) + .unwrap(); + drop(file); + f.reject("duplicate"); +} + +#[test] +fn rejects_overflowing_payload_offsets_without_panicking() { + let f = Fixture::new(); + let elements = usize::MAX / 8; + let mut header = serde_json::Map::new(); + for i in 0..8 { + header.insert( + format!("unrelated.language.{i}.weight"), + json!({ + "dtype": "U8", "shape": [elements], + "data_offsets": [i * elements, (i + 1) * elements] + }), + ); + } + let header = serde_json::to_vec(&header).unwrap(); + let mut file = File::create(f.base.join("model.safetensors")).unwrap(); + file.write_all(&(header.len() as u64).to_le_bytes()) + .unwrap(); + file.write_all(&header).unwrap(); + drop(file); + f.reject("safetensors"); +} From 1512852250fc055d433112eb95f78936dfcc8d15 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Fri, 2 Oct 2026 13:18:13 +0800 Subject: [PATCH 2/2] chore(cua_s1): limit PR diff to core implementation --- recipe/cua_s1/native.md | 3 - recipe/cua_s1/native_vision.md | 64 - src/models/cua_s1/native/Cargo.toml | 4 - .../cua_s1/native/examples/inspect_vision.rs | 22 - tests/cua_s1/fixtures/vision/README.md | 31 - .../fixtures/vision/adapter-tensors.json | 702 ------ .../fixtures/vision/adapter_config.json | 48 - .../cua_s1/fixtures/vision/base-tensors.json | 1887 ----------------- tests/cua_s1/fixtures/vision/config.json | 104 - tests/cua_s1/vision_loader.rs | 466 ---- 10 files changed, 3331 deletions(-) delete mode 100644 recipe/cua_s1/native_vision.md delete mode 100644 src/models/cua_s1/native/examples/inspect_vision.rs delete mode 100644 tests/cua_s1/fixtures/vision/README.md delete mode 100644 tests/cua_s1/fixtures/vision/adapter-tensors.json delete mode 100644 tests/cua_s1/fixtures/vision/adapter_config.json delete mode 100644 tests/cua_s1/fixtures/vision/base-tensors.json delete mode 100644 tests/cua_s1/fixtures/vision/config.json delete mode 100644 tests/cua_s1/vision_loader.rs diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index ba27138..018feff 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -41,6 +41,3 @@ cargo test -p omni-cua-s1-native CUA_S1_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ cargo test --release -p omni-cua-s1-native --test kernels -- --ignored ``` - -For CPU-only structural inspection of the separate vision base weights and -multimodal adapter, see [native vision checkpoint inspection](native_vision.md). diff --git a/recipe/cua_s1/native_vision.md b/recipe/cua_s1/native_vision.md deleted file mode 100644 index 2a51b83..0000000 --- a/recipe/cua_s1/native_vision.md +++ /dev/null @@ -1,64 +0,0 @@ -# Cua-S1 native vision checkpoint inspection - -The model-owned Rust module `omni_cua_s1_native::vision` loads and structurally -validates the Qwen3.5-4B vision weights and Cua-S1 4B 0.2 **multimodal** LoRA on -CPU. It keeps the 297 BF16 base tensors and 100 FP32 adapter tensors separate. -The adapter contains 50 A/B pairs, rank 16, alpha 32, and scale 2. No weights are -merged or converted. This increment does not preprocess images, run the vision -encoder, call CUDA, or enable image requests in the native text worker. - -Prepare checkpoints from these pinned upstream revisions: - -| Checkpoint | Revision | Required files | -| --- | --- | --- | -| `Qwen/Qwen3.5-4B` | `851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a` | `config.json` and `model.safetensors`, or `model.safetensors.index.json` and its vision-bearing shards | -| `cua-ai/cua-s1-4b-0.2` multimodal adapter | `16818868b0cc7813808aae4e87b417657046ab79` | `adapter_config.json` and `adapter_model.safetensors` from the multimodal directory | - -Verify downloaded files against trusted upstream hashes before loading them. -Structural validation checks names, shapes, dtypes, configuration, and shard -mapping; it does **not** establish cryptographic identity or validate tensor -values. A bare base checkpoint or the `text` adapter is insufficient. - -From the repository root, pass exactly the base and multimodal adapter directories: - -```sh -cargo run --locked -p omni-cua-s1-native --example inspect_vision -- \ - weights/Qwen3.5-4B weights/cua-s1-4b-0.2/multimodal -``` - -The example reports the vision configuration, LoRA rank and scale, and tensor -counts. It needs neither a GPU nor a CUDA shared library. The index loader opens -only shards assigned visual tensors; unrelated language tensors in those shards -are ignored. Unexpected visual tensors, incompatible configuration, invalid -safetensors, missing tensors, mismatched index entries, and paths escaping the -checkpoint directory (including symlinks) are rejected. - -`VisionCheckpoint::load(base_dir, adapter_dir)` owns the memory maps and caches -validated tensor metadata. `config()` and `adapter()` expose the configuration; -`base_names()` / `adapter_names()` enumerate visual tensor names, and -`base_tensor(name)` / `adapter_tensor(name)` return borrowed safetensors -`TensorView`s with the original bytes. Views cannot outlive the checkpoint. -Accessors do not parse the file headers again. - -**Do not modify, replace in place, or truncate checkpoint files while a loaded -checkpoint or any of its views exists.** As with the existing native text -loader, callers must ensure that the memory-mapped files remain immutable. - -The supported layout is the pinned 4B configuration: 24 vision blocks, hidden -size 1024, intermediate size 4096, 16 heads, 2304 positions, output width 2560, -3 input channels, patch size 16, temporal patch size 2, spatial merge size 2, -`gelu_pytorch_tanh`, and no DeepStack. Text hidden size must equal vision output -width. LoRA targets both MLP matrices in every vision block and both merger -matrices. Math-changing PEFT options such as DoRA, rsLoRA, biases, target -exclusions, and custom rank/alpha patterns are unsupported. - -Run the CPU integration tests with: - -```sh -cargo test --locked -p omni-cua-s1-native --test vision_loader -``` - -Tests use sparse safetensors with real tensor shapes and small sentinel values; -no weight download or multi-gigabyte in-memory tensor allocation is required. -Their independent metadata oracle and provenance live in -[`tests/cua_s1/fixtures/vision/`](../../tests/cua_s1/fixtures/vision/). diff --git a/src/models/cua_s1/native/Cargo.toml b/src/models/cua_s1/native/Cargo.toml index e6903b7..9bc34e9 100644 --- a/src/models/cua_s1/native/Cargo.toml +++ b/src/models/cua_s1/native/Cargo.toml @@ -22,7 +22,3 @@ serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_ord # the onig regex backend, as in the Python tokenizers wheel tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "sync"] } - -[[test]] -name = "vision_loader" -path = "../../../../tests/cua_s1/vision_loader.rs" diff --git a/src/models/cua_s1/native/examples/inspect_vision.rs b/src/models/cua_s1/native/examples/inspect_vision.rs deleted file mode 100644 index afffb19..0000000 --- a/src/models/cua_s1/native/examples/inspect_vision.rs +++ /dev/null @@ -1,22 +0,0 @@ -use anyhow::{Result, ensure}; -use omni_cua_s1_native::vision::VisionCheckpoint; - -fn main() -> Result<()> { - let args: Vec<_> = std::env::args_os().skip(1).collect(); - ensure!( - args.len() == 2, - "usage: inspect_vision BASE_DIR MULTIMODAL_ADAPTER_DIR" - ); - let checkpoint = VisionCheckpoint::load(&args[0], &args[1])?; - println!("Vision: {:#?}", checkpoint.config()); - println!("Base: {} BF16 tensors", checkpoint.base_names().count()); - println!( - "Adapter: {} FP32 tensors ({} pairs), rank {}, alpha {}, scale {}", - checkpoint.adapter_names().count(), - checkpoint.adapter_names().count() / 2, - checkpoint.adapter().rank, - checkpoint.adapter().alpha, - checkpoint.adapter().scale() - ); - Ok(()) -} diff --git a/tests/cua_s1/fixtures/vision/README.md b/tests/cua_s1/fixtures/vision/README.md deleted file mode 100644 index 21a8ca6..0000000 --- a/tests/cua_s1/fixtures/vision/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Pinned vision metadata fixtures - -These fixtures are an independent test oracle captured from upstream metadata, -not generated by the production Rust inventory builder: - -- Base: `Qwen/Qwen3.5-4B`, revision - `851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a`. -- Adapter: `cua-ai/cua-s1-4b-0.2`, revision - `16818868b0cc7813808aae4e87b417657046ab79`, multimodal adapter. -- `config.json` and `adapter_config.json`: upstream configuration files with LF line endings. -- `base-tensors.json`: visual names, shapes, and dtypes read from the pinned - base safetensors headers (prefix `model.visual.`): 297 BF16 tensors. -- `adapter-tensors.json`: visual names, shapes, and dtypes read from the pinned - multimodal adapter safetensors header (prefix - `base_model.model.model.visual.`): 100 FP32 tensors / 50 LoRA pairs. - -No weight payloads are included. Tests construct sparse files with these shapes -and sentinel values. Production code never reads these fixtures. Fixture -matching proves structural compatibility, not checkpoint identity: verify real -checkpoint downloads against trusted upstream hashes separately. - -Exact pinned sources: - -- [Base configuration](https://huggingface.co/Qwen/Qwen3.5-4B/resolve/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/config.json) -- [Multimodal adapter configuration](https://huggingface.co/cua-ai/cua-s1-4b-0.2/resolve/16818868b0cc7813808aae4e87b417657046ab79/multimodal/adapter_config.json) -- [Base second shard, containing the vision tensors](https://huggingface.co/Qwen/Qwen3.5-4B/resolve/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/model.safetensors-00002-of-00002.safetensors) -- [Multimodal adapter safetensors](https://huggingface.co/cua-ai/cua-s1-4b-0.2/resolve/16818868b0cc7813808aae4e87b417657046ab79/multimodal/adapter_model.safetensors) - -The tensor inventories retain only the `shape` and `dtype` fields of the visual -entries in those safetensors headers; byte offsets and language tensors are not -part of the oracle. diff --git a/tests/cua_s1/fixtures/vision/adapter-tensors.json b/tests/cua_s1/fixtures/vision/adapter-tensors.json deleted file mode 100644 index ac90604..0000000 --- a/tests/cua_s1/fixtures/vision/adapter-tensors.json +++ /dev/null @@ -1,702 +0,0 @@ -{ - "base_model.model.model.visual.blocks.0.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.0.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.0.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.0.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.1.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.1.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.1.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.1.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.10.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.10.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.10.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.10.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.11.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.11.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.11.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.11.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.12.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.12.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.12.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.12.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.13.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.13.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.13.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.13.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.14.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.14.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.14.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.14.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.15.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.15.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.15.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.15.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.16.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.16.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.16.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.16.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.17.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.17.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.17.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.17.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.18.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.18.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.18.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.18.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.19.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.19.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.19.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.19.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.2.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.2.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.2.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.2.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.20.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.20.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.20.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.20.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.21.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.21.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.21.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.21.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.22.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.22.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.22.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.22.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.23.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.23.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.23.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.23.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.3.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.3.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.3.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.3.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.4.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.4.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.4.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.4.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.5.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.5.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.5.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.5.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.6.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.6.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.6.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.6.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.7.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.7.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.7.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.7.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.8.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.8.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.8.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.8.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.blocks.9.mlp.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 1024 - ] - }, - "base_model.model.model.visual.blocks.9.mlp.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.blocks.9.mlp.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.blocks.9.mlp.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 1024, - 16 - ] - }, - "base_model.model.model.visual.merger.linear_fc1.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.merger.linear_fc1.lora_B.weight": { - "dtype": "F32", - "shape": [ - 4096, - 16 - ] - }, - "base_model.model.model.visual.merger.linear_fc2.lora_A.weight": { - "dtype": "F32", - "shape": [ - 16, - 4096 - ] - }, - "base_model.model.model.visual.merger.linear_fc2.lora_B.weight": { - "dtype": "F32", - "shape": [ - 2560, - 16 - ] - } -} diff --git a/tests/cua_s1/fixtures/vision/adapter_config.json b/tests/cua_s1/fixtures/vision/adapter_config.json deleted file mode 100644 index 52b492f..0000000 --- a/tests/cua_s1/fixtures/vision/adapter_config.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "alora_invocation_tokens": null, - "alpha_pattern": {}, - "arrow_config": null, - "auto_mapping": null, - "base_model_name_or_path": "Qwen/Qwen3.5-4B", - "bias": "none", - "corda_config": null, - "ensure_weight_tying": false, - "eva_config": null, - "exclude_modules": null, - "fan_in_fan_out": false, - "inference_mode": true, - "init_lora_weights": true, - "layer_replication": null, - "layers_pattern": null, - "layers_to_transform": null, - "loftq_config": {}, - "lora_alpha": 32, - "lora_bias": false, - "lora_dropout": 0.05, - "megatron_config": null, - "megatron_core": "megatron.core", - "modules_to_save": null, - "peft_type": "LORA", - "peft_version": "0.18.1", - "qalora_group_size": 16, - "r": 16, - "rank_pattern": {}, - "revision": null, - "target_modules": [ - "up_proj", - "k_proj", - "linear_fc1", - "q_proj", - "linear_fc2", - "down_proj", - "gate_proj", - "o_proj", - "v_proj" - ], - "target_parameters": null, - "task_type": "CAUSAL_LM", - "trainable_token_indices": null, - "use_dora": false, - "use_qalora": false, - "use_rslora": false -} diff --git a/tests/cua_s1/fixtures/vision/base-tensors.json b/tests/cua_s1/fixtures/vision/base-tensors.json deleted file mode 100644 index 0c2d0b9..0000000 --- a/tests/cua_s1/fixtures/vision/base-tensors.json +++ /dev/null @@ -1,1887 +0,0 @@ -{ - "model.visual.blocks.0.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.0.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.0.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.0.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.0.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.0.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.0.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.0.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.0.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.0.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.0.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.0.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.1.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.1.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.1.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.1.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.1.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.1.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.1.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.1.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.1.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.1.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.1.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.1.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.10.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.10.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.10.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.10.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.10.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.10.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.10.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.10.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.10.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.10.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.10.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.10.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.11.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.11.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.11.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.11.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.11.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.11.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.11.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.11.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.11.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.11.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.11.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.11.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.12.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.12.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.12.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.12.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.12.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.12.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.12.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.12.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.12.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.12.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.12.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.12.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.13.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.13.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.13.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.13.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.13.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.13.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.13.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.13.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.13.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.13.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.13.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.13.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.14.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.14.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.14.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.14.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.14.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.14.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.14.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.14.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.14.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.14.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.14.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.14.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.15.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.15.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.15.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.15.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.15.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.15.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.15.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.15.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.15.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.15.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.15.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.15.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.16.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.16.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.16.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.16.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.16.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.16.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.16.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.16.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.16.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.16.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.16.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.16.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.17.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.17.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.17.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.17.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.17.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.17.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.17.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.17.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.17.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.17.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.17.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.17.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.18.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.18.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.18.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.18.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.18.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.18.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.18.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.18.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.18.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.18.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.18.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.18.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.19.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.19.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.19.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.19.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.19.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.19.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.19.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.19.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.19.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.19.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.19.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.19.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.2.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.2.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.2.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.2.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.2.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.2.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.2.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.2.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.2.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.2.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.2.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.2.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.20.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.20.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.20.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.20.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.20.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.20.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.20.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.20.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.20.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.20.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.20.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.20.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.21.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.21.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.21.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.21.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.21.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.21.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.21.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.21.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.21.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.21.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.21.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.21.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.22.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.22.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.22.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.22.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.22.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.22.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.22.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.22.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.22.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.22.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.22.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.22.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.23.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.23.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.23.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.23.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.23.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.23.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.23.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.23.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.23.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.23.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.23.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.23.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.3.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.3.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.3.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.3.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.3.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.3.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.3.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.3.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.3.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.3.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.3.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.3.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.4.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.4.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.4.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.4.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.4.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.4.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.4.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.4.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.4.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.4.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.4.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.4.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.5.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.5.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.5.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.5.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.5.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.5.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.5.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.5.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.5.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.5.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.5.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.5.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.6.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.6.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.6.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.6.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.6.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.6.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.6.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.6.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.6.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.6.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.6.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.6.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.7.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.7.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.7.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.7.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.7.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.7.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.7.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.7.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.7.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.7.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.7.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.7.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.8.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.8.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.8.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.8.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.8.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.8.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.8.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.8.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.8.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.8.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.8.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.8.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.9.attn.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.9.attn.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 1024 - ] - }, - "model.visual.blocks.9.attn.qkv.bias": { - "dtype": "BF16", - "shape": [ - 3072 - ] - }, - "model.visual.blocks.9.attn.qkv.weight": { - "dtype": "BF16", - "shape": [ - 3072, - 1024 - ] - }, - "model.visual.blocks.9.mlp.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.blocks.9.mlp.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 1024 - ] - }, - "model.visual.blocks.9.mlp.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.9.mlp.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 4096 - ] - }, - "model.visual.blocks.9.norm1.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.9.norm1.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.9.norm2.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.blocks.9.norm2.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.merger.linear_fc1.bias": { - "dtype": "BF16", - "shape": [ - 4096 - ] - }, - "model.visual.merger.linear_fc1.weight": { - "dtype": "BF16", - "shape": [ - 4096, - 4096 - ] - }, - "model.visual.merger.linear_fc2.bias": { - "dtype": "BF16", - "shape": [ - 2560 - ] - }, - "model.visual.merger.linear_fc2.weight": { - "dtype": "BF16", - "shape": [ - 2560, - 4096 - ] - }, - "model.visual.merger.norm.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.merger.norm.weight": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.patch_embed.proj.bias": { - "dtype": "BF16", - "shape": [ - 1024 - ] - }, - "model.visual.patch_embed.proj.weight": { - "dtype": "BF16", - "shape": [ - 1024, - 3, - 2, - 16, - 16 - ] - }, - "model.visual.pos_embed.weight": { - "dtype": "BF16", - "shape": [ - 2304, - 1024 - ] - } -} diff --git a/tests/cua_s1/fixtures/vision/config.json b/tests/cua_s1/fixtures/vision/config.json deleted file mode 100644 index cca4041..0000000 --- a/tests/cua_s1/fixtures/vision/config.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "architectures": [ - "Qwen3_5ForConditionalGeneration" - ], - "image_token_id": 248056, - "model_type": "qwen3_5", - "text_config": { - "attention_bias": false, - "attention_dropout": 0.0, - "attn_output_gate": true, - "dtype": "bfloat16", - "eos_token_id": 248044, - "full_attention_interval": 4, - "head_dim": 256, - "hidden_act": "silu", - "hidden_size": 2560, - "initializer_range": 0.02, - "intermediate_size": 9216, - "layer_types": [ - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention" - ], - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "linear_value_head_dim": 128, - "max_position_embeddings": 262144, - "mlp_only_layers": [], - "model_type": "qwen3_5_text", - "mtp_num_hidden_layers": 1, - "mtp_use_dedicated_embeddings": false, - "num_attention_heads": 16, - "num_hidden_layers": 32, - "num_key_value_heads": 4, - "rms_norm_eps": 1e-06, - "tie_word_embeddings": true, - "use_cache": true, - "vocab_size": 248320, - "mamba_ssm_dtype": "float32", - "rope_parameters": { - "mrope_interleaved": true, - "mrope_section": [ - 11, - 11, - 10 - ], - "rope_type": "default", - "rope_theta": 10000000, - "partial_rotary_factor": 0.25 - } - }, - "tie_word_embeddings": true, - "transformers_version": "4.57.0.dev0", - "video_token_id": 248057, - "vision_config": { - "deepstack_visual_indexes": [], - "depth": 24, - "hidden_act": "gelu_pytorch_tanh", - "hidden_size": 1024, - "in_channels": 3, - "initializer_range": 0.02, - "intermediate_size": 4096, - "model_type": "qwen3_5", - "num_heads": 16, - "num_position_embeddings": 2304, - "out_hidden_size": 2560, - "patch_size": 16, - "spatial_merge_size": 2, - "temporal_patch_size": 2 - }, - "vision_end_token_id": 248054, - "vision_start_token_id": 248053 -} diff --git a/tests/cua_s1/vision_loader.rs b/tests/cua_s1/vision_loader.rs deleted file mode 100644 index 8de73e2..0000000 --- a/tests/cua_s1/vision_loader.rs +++ /dev/null @@ -1,466 +0,0 @@ -use omni_cua_s1_native::vision::VisionCheckpoint; -use safetensors::Dtype; -use serde_json::{Value, json}; -use std::{ - collections::BTreeMap, - fs::{self, File}, - io::{Seek, SeekFrom, Write}, - path::{Path, PathBuf}, - sync::atomic::{AtomicUsize, Ordering}, -}; - -const BASE: &str = "model.visual."; -const ADAPTER: &str = "base_model.model.model.visual."; -type Inventory = BTreeMap; -fn inventory(adapter: bool) -> Inventory { - serde_json::from_str(if adapter { - include_str!("fixtures/vision/adapter-tensors.json") - } else { - include_str!("fixtures/vision/base-tensors.json") - }) - .unwrap() -} -fn sparse(path: &Path, tensors: &Inventory) { - let mut offset = 0u64; - let mut header = serde_json::Map::new(); - for (name, tensor) in tensors { - let bytes = tensor["shape"] - .as_array() - .unwrap() - .iter() - .map(|n| n.as_u64().unwrap()) - .product::() - * if tensor["dtype"] == "BF16" { 2 } else { 4 }; - header.insert(name.clone(), json!({"dtype":tensor["dtype"],"shape":tensor["shape"],"data_offsets":[offset, offset+bytes]})); - offset += bytes; - } - let header = serde_json::to_vec(&header).unwrap(); - let mut file = File::create(path).unwrap(); - file.write_all(&(header.len() as u64).to_le_bytes()) - .unwrap(); - file.write_all(&header).unwrap(); - file.set_len(8 + header.len() as u64 + offset).unwrap(); - for tensor in serde_json::from_slice::(&header) - .unwrap() - .as_object() - .unwrap() - .values() - { - file.seek(SeekFrom::Start( - 8 + header.len() as u64 + tensor["data_offsets"][0].as_u64().unwrap(), - )) - .unwrap(); - file.write_all(if tensor["dtype"] == "BF16" { - &[0x80, 0x3f] - } else { - &[0, 0, 0x80, 0x3f] - }) - .unwrap(); - } -} -struct Fixture { - root: PathBuf, - base: PathBuf, - adapter: PathBuf, -} -impl Fixture { - fn new() -> Self { - static NEXT: AtomicUsize = AtomicUsize::new(0); - let root = std::env::temp_dir().join(format!( - "cua-vision-{}-{}", - std::process::id(), - NEXT.fetch_add(1, Ordering::Relaxed) - )); - let base = root.join("base"); - let adapter = root.join("adapter"); - fs::create_dir_all(&base).unwrap(); - fs::create_dir_all(&adapter).unwrap(); - fs::write( - base.join("config.json"), - include_str!("fixtures/vision/config.json"), - ) - .unwrap(); - fs::write( - adapter.join("adapter_config.json"), - include_str!("fixtures/vision/adapter_config.json"), - ) - .unwrap(); - sparse(&base.join("model.safetensors"), &inventory(false)); - sparse(&adapter.join("adapter_model.safetensors"), &inventory(true)); - Self { - root, - base, - adapter, - } - } - fn load(&self) -> anyhow::Result { - VisionCheckpoint::load(&self.base, &self.adapter) - } - fn reject(&self, expected: &str) { - let error = match self.load() { - Ok(_) => panic!("accepted invalid checkpoint: {expected}"), - Err(error) => format!("{error:#}"), - }; - assert!( - error.contains(expected), - "expected {expected:?}, got {error}" - ); - } - fn config(&self, adapter: bool, key: &str, value: Value) { - let path = if adapter { - self.adapter.join("adapter_config.json") - } else { - self.base.join("config.json") - }; - let mut config: Value = serde_json::from_str(if adapter { - include_str!("fixtures/vision/adapter_config.json") - } else { - include_str!("fixtures/vision/config.json") - }) - .unwrap(); - if adapter { - config[key] = value; - } else if key == "text_hidden" { - config["text_config"]["hidden_size"] = value; - } else { - config["vision_config"][key] = value; - } - fs::write(path, serde_json::to_vec(&config).unwrap()).unwrap(); - } - fn shards(&self) -> BTreeMap { - let (a, b): (Inventory, Inventory) = inventory(false) - .into_iter() - .partition(|(n, _)| n.contains("blocks.")); - sparse(&self.base.join("a.safetensors"), &a); - sparse(&self.base.join("b.safetensors"), &b); - let mut map: BTreeMap = a - .keys() - .map(|n| (n.clone(), "a.safetensors".into())) - .chain(b.keys().map(|n| (n.clone(), "b.safetensors".into()))) - .collect(); - map.insert( - "model.language_model.weight".into(), - "absent-language.safetensors".into(), - ); - self.index(&map); - fs::remove_file(self.base.join("model.safetensors")).unwrap(); - map - } - fn index(&self, map: &BTreeMap) { - fs::write( - self.base.join("model.safetensors.index.json"), - serde_json::to_vec(&json!({"weight_map":map})).unwrap(), - ) - .unwrap(); - } -} -impl Drop for Fixture { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.root); - } -} -fn check(checkpoint: &VisionCheckpoint) { - assert_eq!(checkpoint.config().depth, 24); - assert_eq!(checkpoint.config().hidden_size, 1024); - assert_eq!(checkpoint.adapter().rank, 16); - assert_eq!(checkpoint.adapter().alpha, 32); - assert_eq!(checkpoint.adapter().scale(), 2.0); - for adapter in [false, true] { - let expected = inventory(adapter); - let names: Vec<_> = if adapter { - checkpoint.adapter_names().collect() - } else { - checkpoint.base_names().collect() - }; - assert_eq!(names.len(), if adapter { 100 } else { 297 }); - for (name, spec) in expected { - let tensor = if adapter { - checkpoint.adapter_tensor(&name) - } else { - checkpoint.base_tensor(&name) - } - .unwrap(); - assert_eq!( - tensor.shape(), - serde_json::from_value::>(spec["shape"].clone()).unwrap() - ); - assert_eq!( - tensor.dtype(), - if adapter { Dtype::F32 } else { Dtype::BF16 } - ); - assert_eq!( - &tensor.data()[..if adapter { 4 } else { 2 }], - if adapter { - &[0, 0, 0x80, 0x3f][..] - } else { - &[0x80, 0x3f][..] - } - ); - } - } - assert!(checkpoint.base_tensor("missing").is_err()); -} -#[test] -fn valid_single_matches_independent_inventory() { - let f = Fixture::new(); - check(&f.load().unwrap()); -} -#[test] -fn valid_shards_skip_language_only_files() { - let f = Fixture::new(); - f.shards(); - check(&f.load().unwrap()); -} -#[test] -fn ignores_language_tensors() { - let f = Fixture::new(); - for adapter in [false, true] { - let mut inv = inventory(adapter); - inv.insert( - "unrelated.language.weight".into(), - json!({"shape":[1],"dtype":"F32"}), - ); - sparse( - &if adapter { - f.adapter.join("adapter_model.safetensors") - } else { - f.base.join("model.safetensors") - }, - &inv, - ); - } - check(&f.load().unwrap()); -} -#[test] -fn rejects_missing_wrong_and_extra_tensors() { - let f = Fixture::new(); - for adapter in [false, true] { - let path = if adapter { - f.adapter.join("adapter_model.safetensors") - } else { - f.base.join("model.safetensors") - }; - let original = inventory(adapter); - let name = original.keys().next().unwrap().clone(); - let mut inv = original.clone(); - inv.remove(&name); - sparse(&path, &inv); - f.reject("missing"); - let mut inv = original.clone(); - inv.get_mut(&name).unwrap()["shape"] = json!([1]); - sparse(&path, &inv); - f.reject("shape"); - let mut inv = original.clone(); - inv.get_mut(&name).unwrap()["dtype"] = json!(if adapter { "BF16" } else { "F32" }); - sparse(&path, &inv); - f.reject("dtype"); - let mut inv = original.clone(); - inv.insert( - format!("{}unexpected.weight", if adapter { ADAPTER } else { BASE }), - json!({"shape":[1],"dtype":"F32"}), - ); - sparse(&path, &inv); - f.reject("unexpected"); - sparse(&path, &original); - } -} -#[test] -fn rejects_truncated_and_malformed_safetensors() { - let f = Fixture::new(); - for adapter in [false, true] { - let path = if adapter { - f.adapter.join("adapter_model.safetensors") - } else { - f.base.join("model.safetensors") - }; - let len = fs::metadata(&path).unwrap().len(); - File::options() - .write(true) - .open(&path) - .unwrap() - .set_len(len - 1) - .unwrap(); - f.reject("safetensors"); - fs::write(&path, b"invalid").unwrap(); - f.reject("safetensors"); - sparse(&path, &inventory(adapter)); - } -} -#[test] -fn rejects_index_missing_extra_misrouting_and_duplicates() { - let f = Fixture::new(); - let original = f.shards(); - let name = inventory(false).keys().next().unwrap().clone(); - let mut map = original.clone(); - map.remove(&name); - f.index(&map); - f.reject("missing"); - let mut map = original.clone(); - map.insert(format!("{BASE}unexpected"), "a.safetensors".into()); - f.index(&map); - f.reject("unexpected"); - let mut map = original.clone(); - map.insert(name.clone(), "b.safetensors".into()); - f.index(&map); - f.reject("index"); - f.index(&original); - let mut b: Inventory = inventory(false) - .into_iter() - .filter(|(n, _)| !n.contains("blocks.")) - .collect(); - b.insert(name.clone(), inventory(false)[&name].clone()); - sparse(&f.base.join("b.safetensors"), &b); - f.reject("index"); - fs::write( - f.base.join("model.safetensors.index.json"), - format!("{{\"weight_map\":{{\"{name}\":\"a.safetensors\",\"{name}\":\"b.safetensors\"}}}}"), - ) - .unwrap(); - f.reject("duplicate"); -} -#[test] -fn rejects_index_paths_outside_directory() { - let f = Fixture::new(); - let original = f.shards(); - let name = inventory(false).keys().next().unwrap().clone(); - for path in ["../outside.safetensors", "/tmp/outside.safetensors"] { - let mut map = original.clone(); - map.insert(name.clone(), path.into()); - f.index(&map); - f.reject("checkpoint directory"); - } -} -#[cfg(unix)] -#[test] -fn rejects_symlink_escape() { - use std::os::unix::fs::symlink; - let f = Fixture::new(); - fs::rename( - f.base.join("model.safetensors"), - f.root.join("outside.safetensors"), - ) - .unwrap(); - symlink( - f.root.join("outside.safetensors"), - f.base.join("model.safetensors"), - ) - .unwrap(); - f.reject("checkpoint directory"); -} -#[test] -fn validates_full_vision_configuration() { - let f = Fixture::new(); - for (key, value) in [ - ("depth", json!(23)), - ("hidden_size", json!(512)), - ("intermediate_size", json!(2048)), - ("num_heads", json!(8)), - ("num_position_embeddings", json!(1024)), - ("out_hidden_size", json!(2048)), - ("in_channels", json!(1)), - ("patch_size", json!(14)), - ("temporal_patch_size", json!(1)), - ("spatial_merge_size", json!(1)), - ("hidden_act", json!("gelu")), - ("deepstack_visual_indexes", json!([1])), - ("text_hidden", json!(2048)), - ] { - f.config(false, key, value); - f.reject("config"); - } -} -#[test] -fn rejects_adapter_math_and_target_changes() { - let f = Fixture::new(); - for (key, value) in [ - ("r", json!(8)), - ("lora_alpha", json!(16)), - ("bias", json!("all")), - ("lora_bias", json!(true)), - ("use_dora", json!(true)), - ("use_rslora", json!(true)), - ("use_qalora", json!(true)), - ("fan_in_fan_out", json!(true)), - ("rank_pattern", json!({"linear_fc1":8})), - ("alpha_pattern", json!({"linear_fc1":16})), - ("exclude_modules", json!(["linear_fc1"])), - ("target_modules", json!(["up_proj", "down_proj"])), - ("modules_to_save", json!(["merger"])), - ("layers_to_transform", json!([1])), - ("target_parameters", json!(["weight"])), - ("layer_replication", json!([[0, 1]])), - ("alora_invocation_tokens", json!([1])), - ("peft_type", json!("IA3")), - ] { - f.config(true, key, value); - f.reject("adapter config"); - } -} -#[test] -fn requires_multimodal_adapter() { - let f = Fixture::new(); - fs::remove_file(f.adapter.join("adapter_model.safetensors")).unwrap(); - f.reject("adapter_model.safetensors"); - sparse( - &f.adapter.join("adapter_model.safetensors"), - &BTreeMap::from([( - "base_model.model.model.language_model.weight".into(), - json!({"shape":[1],"dtype":"F32"}), - )]), - ); - f.reject("missing"); -} - -#[test] -fn rejects_duplicate_visual_keys_in_safetensors_header() { - use std::io::Read; - let f = Fixture::new(); - let path = f.base.join("model.safetensors"); - let mut file = File::open(&path).unwrap(); - let total = file.metadata().unwrap().len(); - let mut size = [0; 8]; - file.read_exact(&mut size).unwrap(); - let old_len = u64::from_le_bytes(size); - let mut header = vec![0; old_len as usize]; - file.read_exact(&mut header).unwrap(); - drop(file); - let entries: BTreeMap = serde_json::from_slice(&header).unwrap(); - let (name, info) = entries.first_key_value().unwrap(); - let mut duplicate = String::from_utf8(header).unwrap(); - duplicate.pop(); - duplicate.push_str(&format!( - ",{}:{info}}}", - serde_json::to_string(name).unwrap() - )); - let mut file = File::create(&path).unwrap(); - file.write_all(&(duplicate.len() as u64).to_le_bytes()) - .unwrap(); - file.write_all(duplicate.as_bytes()).unwrap(); - file.set_len(total - old_len + duplicate.len() as u64) - .unwrap(); - drop(file); - f.reject("duplicate"); -} - -#[test] -fn rejects_overflowing_payload_offsets_without_panicking() { - let f = Fixture::new(); - let elements = usize::MAX / 8; - let mut header = serde_json::Map::new(); - for i in 0..8 { - header.insert( - format!("unrelated.language.{i}.weight"), - json!({ - "dtype": "U8", "shape": [elements], - "data_offsets": [i * elements, (i + 1) * elements] - }), - ); - } - let header = serde_json::to_vec(&header).unwrap(); - let mut file = File::create(f.base.join("model.safetensors")).unwrap(); - file.write_all(&(header.len() as u64).to_le_bytes()) - .unwrap(); - file.write_all(&header).unwrap(); - drop(file); - f.reject("safetensors"); -}