khora_io/asset/
index_builder.rs

1// Copyright 2025 eraflo
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Canonical asset index builder.
16//!
17//! Walks a project's `assets/` directory and produces an [`AssetMetadata`] list
18//! suitable for [`crate::vfs::VirtualFileSystem::new`] — used by the editor to
19//! build an in-memory index at boot, and by the pack builder (Phase 2) to
20//! produce reproducible release archives.
21//!
22//! # Determinism
23//!
24//! Files are sorted by their **forward-slash relative path** (lexicographic on
25//! the UTF-8 string) so the resulting metadata vector — and therefore the
26//! bincode-encoded index — is byte-identical across invocations. This is what
27//! lets CI compare two pack-builder runs and assert reproducibility.
28//!
29//! # UUID stability
30//!
31//! Each asset's [`AssetUUID`] is resolved through the project's
32//! [`AssetIdRegistry`] when one is supplied via [`IndexBuilder::with_registry`],
33//! and otherwise **defaults** to [`AssetUUID::new_v5`] of the **forward-slash
34//! relative path** (e.g. `"textures/wood.png"`). The registry is authoritative:
35//! once an asset has been renamed/moved in the editor its identity is *frozen*
36//! there, so the same file keeps the same UUID across renames. For any asset
37//! without a registry entry the path-derived default applies, which is what
38//! makes pre-registry projects and the pack builder agree by construction (the
39//! pack builder loads the same registry, so dev and release resolve identically).
40
41use crate::asset::dependencies::{extract_dependencies, type_has_dependency_extractor};
42use crate::asset::id_registry::AssetIdRegistry;
43use anyhow::{anyhow, Context, Result};
44use khora_core::asset::{AssetMetadata, AssetSource, AssetUUID};
45use std::{
46    collections::HashMap,
47    path::{Path, PathBuf},
48};
49
50/// Returns the canonical asset type name for a file extension.
51///
52/// The well-known set is the **single source of truth** shared by:
53/// - the editor's in-memory index (this module),
54/// - the asset browser's tile categorization,
55/// - the pack builder (Phase 2).
56///
57/// All names are lower-case to match the existing decoder registrations
58/// (`crates/khora-agents/tests/asset_loading_test.rs:162` registers
59/// `"texture"`).
60///
61/// Unknown extensions return `Some(<extension>)` rather than `None` —
62/// the engine tracks **everything** under `assets/` regardless of
63/// whether it has a dedicated decoder yet. New file kinds show up in
64/// the asset browser automatically; adding a decoder only changes how
65/// they're consumed at runtime, not whether they enter the VFS.
66pub fn asset_type_for_extension(ext: &str) -> Option<String> {
67    let lower = ext.to_ascii_lowercase();
68    let canonical: Option<&'static str> = match lower.as_str() {
69        // Mesh formats
70        "gltf" | "glb" | "obj" | "fbx" => Some("mesh"),
71        // Texture formats
72        "png" | "jpg" | "jpeg" | "tga" | "bmp" | "hdr" => Some("texture"),
73        // Audio formats
74        "wav" | "ogg" | "mp3" | "flac" => Some("audio"),
75        // Shader formats
76        "wgsl" | "hlsl" | "glsl" => Some("shader"),
77        // Font formats
78        "ttf" | "otf" => Some("font"),
79        // Scene formats (Khora scene files)
80        "kscene" | "scene" => Some("scene"),
81        // Material formats
82        "kmat" | "mat" => Some("material"),
83        // Script formats — data, hot-reloadable, future custom language
84        "kscript" => Some("script"),
85        // Prefab formats (Phase 5 — instanced via SerializationService)
86        "kprefab" => Some("prefab"),
87        _ => None,
88    };
89    Some(canonical.map(|s| s.to_string()).unwrap_or(lower))
90}
91
92/// Asset type assigned to files with no extension at all. These still
93/// enter the VFS so they're visible in the asset browser, but the
94/// engine has no way to dispatch a decoder until the user renames or
95/// re-classifies them.
96pub const EXTENSIONLESS_ASSET_TYPE: &str = "blob";
97
98/// File-name prefixes the scan ignores outright. These are OS / editor
99/// scratch files that should never be part of the project.
100const SCAN_IGNORE_PREFIXES: &[&str] = &[".", "~"];
101
102/// File-name suffixes the scan ignores outright. Editor swap files,
103/// build-tool temporaries, etc.
104const SCAN_IGNORE_SUFFIXES: &[&str] = &[".tmp", ".swp", ".bak", "~"];
105
106/// `true` if a file with the given name should be excluded from the
107/// VFS scan and from hot-reload events. Catches OS scratch files
108/// (`.DS_Store`, `.gitkeep`, …) and editor swap / backup files
109/// (`*.tmp`, `*.swp`, `*.bak`, `*~`).
110pub fn should_skip_file(name: &str) -> bool {
111    if name.is_empty() {
112        return true;
113    }
114    if SCAN_IGNORE_PREFIXES
115        .iter()
116        .any(|prefix| name.starts_with(prefix))
117    {
118        return true;
119    }
120    if SCAN_IGNORE_SUFFIXES
121        .iter()
122        .any(|suffix| name.ends_with(suffix))
123    {
124        return true;
125    }
126    false
127}
128
129/// Recursive scanner that turns a project's `assets/` directory into an
130/// `AssetMetadata` list ready for the VFS.
131///
132/// See module documentation for determinism and UUID stability guarantees.
133pub struct IndexBuilder<'a> {
134    assets_root: &'a Path,
135    registry: Option<&'a AssetIdRegistry>,
136}
137
138impl<'a> IndexBuilder<'a> {
139    /// Creates a new builder rooted at `assets_root`.
140    ///
141    /// `assets_root` must be the **assets directory of a project**, not the
142    /// project root — the relative paths recorded in `AssetMetadata` (and
143    /// hence the UUIDs) are computed relative to it.
144    ///
145    /// Without [`Self::with_registry`], UUIDs are the path-derived
146    /// `new_v5` default (the pre-registry behaviour).
147    pub fn new(assets_root: &'a Path) -> Self {
148        Self {
149            assets_root,
150            registry: None,
151        }
152    }
153
154    /// Resolves each asset's UUID through `registry` (frozen identities win;
155    /// unfrozen paths still fall back to `new_v5`). Supply the same registry in
156    /// dev (editor VFS) and release (pack builder) so UUIDs match by
157    /// construction.
158    pub fn with_registry(mut self, registry: &'a AssetIdRegistry) -> Self {
159        self.registry = Some(registry);
160        self
161    }
162
163    /// Walks the assets root and produces a sorted, deterministic
164    /// `Vec<AssetMetadata>`.
165    ///
166    /// Returns an empty vector if `assets_root` doesn't exist — the editor
167    /// tolerates fresh projects whose `assets/` directory hasn't been
168    /// populated yet.
169    pub fn build_metadata(&self) -> Result<Vec<AssetMetadata>> {
170        if !self.assets_root.exists() {
171            return Ok(Vec::new());
172        }
173
174        // The absolute path is retained alongside the relative one so handled
175        // asset types can have their bytes read for dependency extraction after
176        // sorting. Leaf types are never read (see below).
177        let mut entries: Vec<(String, PathBuf, PathBuf, String)> = Vec::new();
178
179        for entry in walkdir::WalkDir::new(self.assets_root)
180            .follow_links(false)
181            .into_iter()
182            .filter_map(|e| e.ok())
183        {
184            if !entry.file_type().is_file() {
185                continue;
186            }
187            let abs = entry.path();
188            let rel = match abs.strip_prefix(self.assets_root) {
189                Ok(r) => r.to_path_buf(),
190                Err(_) => continue,
191            };
192            let file_name = abs.file_name().and_then(|n| n.to_str()).unwrap_or("");
193            if should_skip_file(file_name) {
194                continue;
195            }
196            let type_name = match rel.extension().and_then(|e| e.to_str()) {
197                Some(ext) => asset_type_for_extension(ext)
198                    .unwrap_or_else(|| EXTENSIONLESS_ASSET_TYPE.to_string()),
199                None => EXTENSIONLESS_ASSET_TYPE.to_string(),
200            };
201            let rel_fwd = rel_to_forward_slash(&rel);
202            entries.push((rel_fwd, rel, abs.to_path_buf(), type_name));
203        }
204
205        // Sort by forward-slash relative path for byte-deterministic output.
206        // Dependencies are extracted *after* sorting so the order they are
207        // visited never affects the result.
208        entries.sort_by(|a, b| a.0.cmp(&b.0));
209
210        let mut metadata = Vec::with_capacity(entries.len());
211        for (rel_fwd, rel_path, abs_path, type_name) in entries {
212            // Registry-frozen identity if present, else the path-derived default.
213            let uuid = match self.registry {
214                Some(reg) => reg.resolve(&rel_fwd),
215                None => AssetUUID::new_v5(&rel_fwd),
216            };
217            // Only read file contents for types whose references we can parse.
218            // Textures, audio, meshes, etc. are never read — this preserves the
219            // builder's stat-only fast path and avoids a per-file I/O cliff.
220            let dependencies = if type_has_dependency_extractor(&type_name) {
221                match std::fs::read(&abs_path) {
222                    Ok(bytes) => extract_dependencies(&type_name, &bytes),
223                    Err(e) => {
224                        log::warn!(
225                            "asset index: failed to read '{}' for dependency extraction: {e}",
226                            rel_fwd
227                        );
228                        Vec::new()
229                    }
230                }
231            } else {
232                Vec::new()
233            };
234            let mut variants = HashMap::with_capacity(1);
235            variants.insert("default".to_string(), AssetSource::Path(rel_path.clone()));
236            metadata.push(AssetMetadata {
237                uuid,
238                source_path: rel_path,
239                asset_type_name: type_name,
240                dependencies,
241                variants,
242                tags: Vec::new(),
243            });
244        }
245        Ok(metadata)
246    }
247
248    /// Convenience: builds metadata and bincode-encodes it into the byte
249    /// vector that [`crate::vfs::VirtualFileSystem::new`] expects.
250    pub fn build_index_bytes(&self) -> Result<Vec<u8>> {
251        let metadata = self.build_metadata()?;
252        let cfg = bincode::config::standard();
253        bincode::serde::encode_to_vec(&metadata, cfg)
254            .map_err(|e| anyhow!("Failed to encode asset index: {}", e))
255            .context("IndexBuilder::build_index_bytes")
256    }
257}
258
259/// Normalizes a relative path to forward-slash separators so UUIDs are
260/// platform-agnostic (`textures\foo.png` on Windows would otherwise hash
261/// differently from `textures/foo.png` on Linux).
262fn rel_to_forward_slash(rel: &Path) -> String {
263    rel.components()
264        .map(|c| c.as_os_str().to_string_lossy().into_owned())
265        .collect::<Vec<_>>()
266        .join("/")
267}
268
269#[cfg(test)]
270mod tests {
271    use super::*;
272    use crate::vfs::VirtualFileSystem;
273    use std::fs;
274    use tempfile::tempdir;
275
276    #[test]
277    fn ext_canonical_mapping() {
278        assert_eq!(asset_type_for_extension("png").as_deref(), Some("texture"));
279        assert_eq!(asset_type_for_extension("PNG").as_deref(), Some("texture"));
280        assert_eq!(asset_type_for_extension("gltf").as_deref(), Some("mesh"));
281        assert_eq!(asset_type_for_extension("kscene").as_deref(), Some("scene"));
282        assert_eq!(
283            asset_type_for_extension("kscript").as_deref(),
284            Some("script")
285        );
286        // Unknown extensions fall back to a per-extension bucket so the
287        // file is still tracked (vs. silently dropped).
288        assert_eq!(asset_type_for_extension("xyz").as_deref(), Some("xyz"));
289        assert_eq!(asset_type_for_extension("MD").as_deref(), Some("md"));
290    }
291
292    #[test]
293    fn index_builder_honours_frozen_registry_uuid() {
294        let dir = tempdir().unwrap();
295        let root = dir.path();
296        fs::create_dir_all(root.join("meshes")).unwrap();
297        fs::write(root.join("meshes").join("hero.gltf"), b"GLTF").unwrap();
298
299        // Default (no registry): UUID is derived from the path.
300        let default_uuid = AssetUUID::new_v5("meshes/hero.gltf");
301        let md = IndexBuilder::new(root).build_metadata().unwrap();
302        assert_eq!(md[0].uuid, default_uuid);
303
304        // Freeze a *different* identity for that path — exactly what a rename
305        // does (the file kept the UUID it had under its previous name).
306        let mut reg = AssetIdRegistry::load(root);
307        let frozen = AssetUUID::new_v5("meshes/protagonist.gltf");
308        reg.freeze("meshes/hero.gltf", frozen);
309
310        let md2 = IndexBuilder::new(root)
311            .with_registry(&reg)
312            .build_metadata()
313            .unwrap();
314        assert_eq!(
315            md2[0].uuid, frozen,
316            "IndexBuilder must resolve the frozen registry UUID, not new_v5(path)"
317        );
318        assert_ne!(md2[0].uuid, default_uuid);
319    }
320
321    #[test]
322    fn build_metadata_tracks_every_file_under_assets() {
323        let dir = tempdir().unwrap();
324        let root = dir.path();
325        fs::create_dir_all(root.join("textures")).unwrap();
326        fs::create_dir_all(root.join("scenes")).unwrap();
327        fs::create_dir_all(root.join("docs")).unwrap();
328        fs::write(root.join("textures").join("foo.png"), b"PNG").unwrap();
329        fs::write(root.join("scenes").join("default.kscene"), b"SCN").unwrap();
330        fs::write(root.join("docs").join("README.md"), b"hello").unwrap();
331        fs::write(root.join("notes"), b"raw").unwrap();
332        // Editor / OS scratch — these MUST still be skipped.
333        fs::write(root.join(".DS_Store"), b"junk").unwrap();
334        fs::write(root.join("textures").join("foo.png.tmp"), b"junk").unwrap();
335        fs::write(root.join("textures").join("foo.png~"), b"junk").unwrap();
336
337        let metadata = IndexBuilder::new(root).build_metadata().unwrap();
338        let by_path: std::collections::HashMap<String, String> = metadata
339            .iter()
340            .map(|m| {
341                (
342                    rel_to_forward_slash(&m.source_path),
343                    m.asset_type_name.clone(),
344                )
345            })
346            .collect();
347
348        assert_eq!(
349            by_path.get("textures/foo.png").map(String::as_str),
350            Some("texture")
351        );
352        assert_eq!(
353            by_path.get("scenes/default.kscene").map(String::as_str),
354            Some("scene")
355        );
356        // Unknown extension still flows into the index.
357        assert_eq!(
358            by_path.get("docs/README.md").map(String::as_str),
359            Some("md")
360        );
361        // No-extension file too — bucketed as "blob".
362        assert_eq!(by_path.get("notes").map(String::as_str), Some("blob"));
363        // Scratch files dropped.
364        assert!(!by_path.contains_key(".DS_Store"));
365        assert!(!by_path.contains_key("textures/foo.png.tmp"));
366        assert!(!by_path.contains_key("textures/foo.png~"));
367        assert_eq!(metadata.len(), 4);
368    }
369
370    #[test]
371    fn build_metadata_is_deterministic() {
372        let dir = tempdir().unwrap();
373        let root = dir.path();
374        fs::create_dir_all(root.join("textures")).unwrap();
375        fs::create_dir_all(root.join("audio")).unwrap();
376        fs::write(root.join("textures").join("a.png"), b"a").unwrap();
377        fs::write(root.join("textures").join("b.png"), b"b").unwrap();
378        fs::write(root.join("audio").join("c.wav"), b"c").unwrap();
379
380        let bytes_a = IndexBuilder::new(root).build_index_bytes().unwrap();
381        let bytes_b = IndexBuilder::new(root).build_index_bytes().unwrap();
382        assert_eq!(
383            bytes_a, bytes_b,
384            "two consecutive builds must be byte-equal"
385        );
386    }
387
388    #[test]
389    fn build_index_bytes_round_trip_through_vfs() {
390        let dir = tempdir().unwrap();
391        let root = dir.path();
392        fs::create_dir_all(root.join("textures")).unwrap();
393        fs::write(root.join("textures").join("foo.png"), b"PNG").unwrap();
394
395        let bytes = IndexBuilder::new(root).build_index_bytes().unwrap();
396        let vfs = VirtualFileSystem::new(&bytes).expect("VFS must decode our index");
397        assert_eq!(vfs.asset_count(), 1);
398        let uuid = AssetUUID::new_v5("textures/foo.png");
399        let meta = vfs.get_metadata(&uuid).expect("VFS must surface the asset");
400        assert_eq!(meta.asset_type_name, "texture");
401    }
402
403    /// Writes a `StandardMaterial` to `path` as `.kmat` RON — the on-disk form
404    /// the index builder reads when extracting dependencies.
405    fn write_kmat(path: &Path, material: &khora_core::asset::StandardMaterial) {
406        let json =
407            khora_data::ecs::material_to_json(material).expect("material serializes to JSON");
408        let ron = ron::ser::to_string(&json).expect("material JSON encodes to RON");
409        fs::write(path, ron).unwrap();
410    }
411
412    #[test]
413    fn material_metadata_lists_its_textures_sorted_and_deduped() {
414        let dir = tempdir().unwrap();
415        let root = dir.path();
416        fs::create_dir_all(root.join("textures")).unwrap();
417        fs::create_dir_all(root.join("materials")).unwrap();
418        fs::write(root.join("textures").join("base.png"), b"PNG").unwrap();
419        fs::write(root.join("textures").join("normal.png"), b"PNG").unwrap();
420
421        // The UUIDs a material stores are derived from the texture's
422        // forward-slash relative path — identical to the UUIDs the index
423        // assigns those texture files.
424        let base_uuid = AssetUUID::new_v5("textures/base.png");
425        let normal_uuid = AssetUUID::new_v5("textures/normal.png");
426        let material = khora_core::asset::StandardMaterial {
427            base_color_texture: Some(base_uuid),
428            // Reuse the base texture in a second slot to exercise dedup.
429            metallic_roughness_texture: Some(base_uuid),
430            normal_map: Some(normal_uuid),
431            ..Default::default()
432        };
433        write_kmat(&root.join("materials").join("wood.kmat"), &material);
434
435        let metadata = IndexBuilder::new(root).build_metadata().unwrap();
436        let mat_meta = metadata
437            .iter()
438            .find(|m| m.asset_type_name == "material")
439            .expect("the material must be indexed");
440
441        let mut expected = vec![base_uuid, normal_uuid];
442        expected.sort();
443        assert_eq!(
444            mat_meta.dependencies, expected,
445            "material deps must be both textures, sorted and deduped"
446        );
447
448        // The texture entries themselves must carry no dependencies — they are
449        // leaf assets and are never read/parsed.
450        for tex in metadata.iter().filter(|m| m.asset_type_name == "texture") {
451            assert!(
452                tex.dependencies.is_empty(),
453                "texture '{}' must have no dependencies",
454                rel_to_forward_slash(&tex.source_path)
455            );
456        }
457    }
458
459    #[test]
460    fn build_index_bytes_is_byte_equal_with_material_and_textures() {
461        let dir = tempdir().unwrap();
462        let root = dir.path();
463        fs::create_dir_all(root.join("textures")).unwrap();
464        fs::create_dir_all(root.join("materials")).unwrap();
465        fs::write(root.join("textures").join("a.png"), b"A").unwrap();
466        fs::write(root.join("textures").join("b.png"), b"B").unwrap();
467
468        let material = khora_core::asset::StandardMaterial {
469            base_color_texture: Some(AssetUUID::new_v5("textures/a.png")),
470            normal_map: Some(AssetUUID::new_v5("textures/b.png")),
471            ..Default::default()
472        };
473        write_kmat(&root.join("materials").join("m.kmat"), &material);
474
475        let bytes_a = IndexBuilder::new(root).build_index_bytes().unwrap();
476        let bytes_b = IndexBuilder::new(root).build_index_bytes().unwrap();
477        assert_eq!(
478            bytes_a, bytes_b,
479            "an index over a material + textures must be byte-deterministic"
480        );
481    }
482
483    #[test]
484    fn corrupt_material_still_builds_with_empty_deps() {
485        let dir = tempdir().unwrap();
486        let root = dir.path();
487        fs::create_dir_all(root.join("materials")).unwrap();
488        fs::write(
489            root.join("materials").join("broken.kmat"),
490            b"this is not valid kmat RON",
491        )
492        .unwrap();
493
494        let metadata = IndexBuilder::new(root)
495            .build_metadata()
496            .expect("a corrupt material must not abort the index build");
497        let mat_meta = metadata
498            .iter()
499            .find(|m| m.asset_type_name == "material")
500            .expect("the corrupt material is still indexed");
501        assert!(
502            mat_meta.dependencies.is_empty(),
503            "a corrupt material yields empty deps, not a panic"
504        );
505    }
506
507    #[test]
508    fn build_metadata_empty_root_returns_empty_vec() {
509        let dir = tempdir().unwrap();
510        let metadata = IndexBuilder::new(dir.path()).build_metadata().unwrap();
511        assert!(metadata.is_empty());
512    }
513
514    #[test]
515    fn build_metadata_missing_root_returns_empty_vec() {
516        let metadata = IndexBuilder::new(Path::new("/__nonexistent__"))
517            .build_metadata()
518            .unwrap();
519        assert!(metadata.is_empty());
520    }
521}
khora_io/asset/index_builder.rs

khora_io/asset/
index_builder.rs
