khora_io/asset/
pack_builder.rs

1// Copyright 2025 eraflo
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Asset pack builder.
16//!
17//! Bundles a project's `assets/` directory into the two-file release layout
18//! consumed by [`crate::asset::PackLoader`]:
19//!
20//! - `<dest>/data.pack` — concatenation of every asset's bytes, in the order
21//!   produced by [`crate::asset::IndexBuilder`] (sorted by forward-slash
22//!   relative path → deterministic).
23//! - `<dest>/index.bin` — `bincode`-encoded `Vec<AssetMetadata>` with each
24//!   metadata's `variants["default"]` rewritten from `AssetSource::Path(rel)`
25//!   to `AssetSource::Packed { offset, size }`.
26//!
27//! UUIDs are produced by `IndexBuilder`, resolved through the project's
28//! [`crate::asset::AssetIdRegistry`] (loaded from `<project>/.khora/`), so they
29//! match what the editor sees during dev mode — same project, same UUIDs in dev
30//! and release, by construction. Projects without a registry fall back to the
31//! path-derived default, which is identical on both sides.
32//!
33//! # Determinism
34//!
35//! Two consecutive `build()` calls on the same source tree produce
36//! byte-identical `index.bin` and `data.pack`. CI relies on this for
37//! reproducibility checks.
38//!
39//! # Progress
40//!
41//! Pass a `crossbeam_channel::Sender<PackProgress>` via [`PackBuilder::with_progress`]
42//! to receive incremental events. The build thread sends one event per file
43//! processed plus a final `Finished` event. The editor's Build dialog drives
44//! its progress bar off this channel.
45
46use anyhow::{anyhow, Context, Result};
47use crossbeam_channel::Sender;
48use khora_core::asset::{AssetSource, CompressionKind};
49use std::{
50    fs::File,
51    io::Write,
52    path::{Path, PathBuf},
53};
54
55use super::{
56    AssetIdRegistry, IndexBuilder, PackLoader, PackManifest, PACK_FLAG_LZ4, PACK_FLAG_MANIFEST,
57};
58
59/// One step of a pack build, suitable for driving a UI progress bar.
60#[derive(Debug, Clone)]
61pub enum PackProgress {
62    /// A new asset is about to be streamed into `data.pack`.
63    Started {
64        /// Forward-slash relative path of the asset being processed.
65        rel_path: String,
66        /// 0-based index of this file in the build.
67        current: usize,
68        /// Total number of files in the build (constant across events).
69        total: usize,
70    },
71    /// The build finished successfully.
72    Finished {
73        /// Number of assets packed.
74        asset_count: usize,
75        /// Total size of `data.pack` in bytes.
76        pack_bytes: u64,
77    },
78    /// The build failed mid-way. The error message is best-effort — the
79    /// caller should still surface the `Result` returned from `build()`.
80    Failed {
81        /// Human-readable description of the failure.
82        message: String,
83    },
84}
85
86/// Output of a successful [`PackBuilder::build`].
87#[derive(Debug, Clone)]
88pub struct PackOutput {
89    /// Absolute path to the produced `index.bin`.
90    pub index_bin: PathBuf,
91    /// Absolute path to the produced `data.pack`.
92    pub data_pack: PathBuf,
93    /// Absolute path to the produced `manifest.bin` (BLAKE3 integrity
94    /// records). `None` when manifest emission was disabled.
95    pub manifest_bin: Option<PathBuf>,
96    /// Number of assets indexed (mirrors `index.bin`'s entry count).
97    pub asset_count: usize,
98    /// Total size of `data.pack` in bytes.
99    pack_bytes: u64,
100}
101
102impl PackOutput {
103    /// Total size of `data.pack` in bytes.
104    pub fn pack_bytes(&self) -> u64 {
105        self.pack_bytes
106    }
107}
108
109/// Builds a release pack from a project's `assets/` directory.
110///
111/// ```ignore
112/// use khora_io::asset::{PackBuilder, PackProgress};
113///
114/// let (tx, rx) = crossbeam_channel::unbounded::<PackProgress>();
115/// std::thread::spawn(move || {
116///     while let Ok(ev) = rx.recv() { /* update UI */ }
117/// });
118///
119/// let out = PackBuilder::new(&project_assets_dir, &dest_dir)
120///     .with_progress(tx)
121///     .build()?;
122/// ```
123pub struct PackBuilder<'a> {
124    assets_root: &'a Path,
125    dest_dir: &'a Path,
126    progress: Option<Sender<PackProgress>>,
127    compression: CompressionKind,
128    write_manifest: bool,
129}
130
131impl<'a> PackBuilder<'a> {
132    /// Creates a new builder. `assets_root` is the project's `assets/`
133    /// directory (the same path that `ProjectVfs` watches in dev mode).
134    /// `dest_dir` is where `index.bin` + `data.pack` will be written —
135    /// created if missing.
136    pub fn new(assets_root: &'a Path, dest_dir: &'a Path) -> Self {
137        Self {
138            assets_root,
139            dest_dir,
140            progress: None,
141            compression: CompressionKind::None,
142            write_manifest: false,
143        }
144    }
145
146    /// Forwards incremental progress to `tx`. Drop the receiver to silently
147    /// disable progress (the sends are best-effort and never block).
148    pub fn with_progress(mut self, tx: Sender<PackProgress>) -> Self {
149        self.progress = Some(tx);
150        self
151    }
152
153    /// Sets the per-entry compression scheme. Default is
154    /// [`CompressionKind::None`]. When set to LZ4, every asset is
155    /// compressed individually so the runtime can decompress on demand
156    /// (no global state, no streaming gzip).
157    pub fn with_compression(mut self, compression: CompressionKind) -> Self {
158        self.compression = compression;
159        self
160    }
161
162    /// Emits a `manifest.bin` sidecar with BLAKE3 hashes of every
163    /// uncompressed asset. The runtime opts in via
164    /// `RuntimeConfig::verify_integrity` to detect corruption /
165    /// tampering at load time.
166    pub fn with_manifest(mut self, enable: bool) -> Self {
167        self.write_manifest = enable;
168        self
169    }
170
171    /// Performs the build. Side effects:
172    ///
173    /// - creates `<dest_dir>/` if missing,
174    /// - writes `<dest_dir>/data.pack` (concatenation of asset bytes),
175    /// - writes `<dest_dir>/index.bin` (`bincode`-encoded metadata).
176    ///
177    /// The two files together are everything `khora_runtime` needs to load
178    /// the project at runtime via [`crate::asset::PackLoader`].
179    pub fn build(self) -> Result<PackOutput> {
180        std::fs::create_dir_all(self.dest_dir).with_context(|| {
181            format!(
182                "Failed to create pack destination: {}",
183                self.dest_dir.display()
184            )
185        })?;
186
187        // Resolve UUIDs through the same identity registry the editor uses so a
188        // renamed asset keeps its frozen UUID in the release pack. The registry
189        // lives at the project root (parent of `assets/`); a missing file yields
190        // an empty registry, i.e. the path-derived default — identical to dev.
191        let project_root = self.assets_root.parent().unwrap_or(self.assets_root);
192        let registry = AssetIdRegistry::load(project_root);
193        let mut metadata = IndexBuilder::new(self.assets_root)
194            .with_registry(&registry)
195            .build_metadata()
196            .context("Pack: failed to build asset metadata")?;
197        let total = metadata.len();
198
199        let data_pack_path = self.dest_dir.join("data.pack");
200        let mut data_file = File::create(&data_pack_path)
201            .with_context(|| format!("Failed to create {}", data_pack_path.display()))?;
202
203        // Write the leading 24-byte header. Asset offsets recorded below
204        // are relative to the start of the asset region (i.e. don't
205        // include the header) — `PackLoader` adds `PACK_HEADER_SIZE` when
206        // seeking. This keeps the index oblivious to the on-disk framing.
207        let mut flags = 0u32;
208        if self.compression == CompressionKind::Lz4 {
209            flags |= PACK_FLAG_LZ4;
210        }
211        if self.write_manifest {
212            flags |= PACK_FLAG_MANIFEST;
213        }
214        PackLoader::write_header(&mut data_file, total as u32, flags)
215            .context("Failed to write pack header")?;
216
217        let mut manifest = if self.write_manifest {
218            Some(PackManifest::new())
219        } else {
220            None
221        };
222
223        let mut offset = 0u64;
224        for (idx, meta) in metadata.iter_mut().enumerate() {
225            // The default variant is `AssetSource::Path(rel)` from
226            // IndexBuilder — extract the rel path before we overwrite it.
227            let rel_path = match meta.variants.get("default") {
228                Some(AssetSource::Path(p)) => p.clone(),
229                Some(AssetSource::Packed { .. }) => {
230                    return Err(anyhow!(
231                        "Pack: asset {:?} already has a Packed variant; \
232                         IndexBuilder must produce Path variants",
233                        meta.uuid
234                    ));
235                }
236                None => {
237                    return Err(anyhow!(
238                        "Pack: asset {:?} has no 'default' variant",
239                        meta.uuid
240                    ));
241                }
242            };
243
244            let rel_str = rel_to_forward_slash(&rel_path);
245            self.send_progress(PackProgress::Started {
246                rel_path: rel_str.clone(),
247                current: idx,
248                total,
249            });
250
251            let abs = self.assets_root.join(&rel_path);
252            let raw = std::fs::read(&abs)
253                .with_context(|| format!("Failed to read asset bytes: {}", abs.display()))?;
254            let uncompressed_size = raw.len() as u64;
255
256            // Manifest hashes uncompressed bytes — the runtime computes
257            // BLAKE3 after decompression so the check is independent of
258            // the compression algorithm.
259            if let Some(m) = manifest.as_mut() {
260                m.insert(meta.uuid, &raw);
261            }
262
263            let (bytes_to_write, compression) = match self.compression {
264                CompressionKind::None => (raw, CompressionKind::None),
265                CompressionKind::Lz4 => {
266                    let compressed = lz4_flex::block::compress(&raw);
267                    // Skip compression when it makes the entry larger
268                    // (already-compressed media: PNG, OGG, FBX).
269                    if compressed.len() < raw.len() {
270                        (compressed, CompressionKind::Lz4)
271                    } else {
272                        (raw, CompressionKind::None)
273                    }
274                }
275            };
276            let on_disk_size = bytes_to_write.len() as u64;
277
278            data_file
279                .write_all(&bytes_to_write)
280                .with_context(|| format!("Failed to append {} to data.pack", rel_str))?;
281
282            meta.variants.insert(
283                "default".to_string(),
284                AssetSource::Packed {
285                    offset,
286                    size: on_disk_size,
287                    uncompressed_size,
288                    compression,
289                },
290            );
291
292            offset += on_disk_size;
293        }
294
295        // Flush before writing index — index references offsets into the
296        // pack we just produced.
297        data_file.flush().context("Failed to flush data.pack")?;
298        drop(data_file);
299
300        let index_bin_path = self.dest_dir.join("index.bin");
301        let cfg = bincode::config::standard();
302        let encoded = bincode::serde::encode_to_vec(&metadata, cfg)
303            .map_err(|e| anyhow!("Failed to encode index: {}", e))?;
304        std::fs::write(&index_bin_path, &encoded)
305            .with_context(|| format!("Failed to write {}", index_bin_path.display()))?;
306
307        let manifest_bin = if let Some(m) = manifest {
308            let manifest_path = self.dest_dir.join("manifest.bin");
309            let manifest_bytes = m.encode().context("Failed to encode manifest")?;
310            std::fs::write(&manifest_path, &manifest_bytes)
311                .with_context(|| format!("Failed to write {}", manifest_path.display()))?;
312            Some(manifest_path)
313        } else {
314            None
315        };
316
317        let pack_bytes = offset;
318        self.send_progress(PackProgress::Finished {
319            asset_count: total,
320            pack_bytes,
321        });
322
323        Ok(PackOutput {
324            index_bin: index_bin_path,
325            data_pack: data_pack_path,
326            manifest_bin,
327            asset_count: total,
328            pack_bytes,
329        })
330    }
331
332    fn send_progress(&self, ev: PackProgress) {
333        if let Some(tx) = &self.progress {
334            // Best-effort — disconnected receiver is fine.
335            let _ = tx.send(ev);
336        }
337    }
338}
339
340fn rel_to_forward_slash(rel: &Path) -> String {
341    rel.components()
342        .map(|c| c.as_os_str().to_string_lossy().into_owned())
343        .collect::<Vec<_>>()
344        .join("/")
345}
346
347#[cfg(test)]
348mod tests {
349    use super::*;
350    use crate::asset::{AssetService, PackLoader};
351    use crate::vfs::VirtualFileSystem;
352    use khora_core::asset::AssetUUID;
353    use khora_telemetry::MetricsRegistry;
354    use std::fs;
355    use std::sync::Arc;
356    use tempfile::tempdir;
357
358    fn seed_project(root: &Path) {
359        fs::create_dir_all(root.join("textures")).unwrap();
360        fs::create_dir_all(root.join("scenes")).unwrap();
361        fs::write(root.join("textures").join("a.png"), b"PNG-A").unwrap();
362        fs::write(root.join("textures").join("b.png"), b"PNG-B").unwrap();
363        fs::write(root.join("scenes").join("default.kscene"), b"SCENE").unwrap();
364    }
365
366    #[test]
367    fn build_produces_index_and_pack() {
368        let proj = tempdir().unwrap();
369        let dest = tempdir().unwrap();
370        seed_project(proj.path());
371
372        let out = PackBuilder::new(proj.path(), dest.path()).build().unwrap();
373
374        assert!(out.index_bin.exists());
375        assert!(out.data_pack.exists());
376        assert_eq!(out.asset_count, 3);
377        assert_eq!(out.pack_bytes(), 5 + 5 + 5); // PNG-A + PNG-B + SCENE
378    }
379
380    #[test]
381    fn build_is_deterministic() {
382        let proj = tempdir().unwrap();
383        let dest_a = tempdir().unwrap();
384        let dest_b = tempdir().unwrap();
385        seed_project(proj.path());
386
387        let _ = PackBuilder::new(proj.path(), dest_a.path())
388            .build()
389            .unwrap();
390        let _ = PackBuilder::new(proj.path(), dest_b.path())
391            .build()
392            .unwrap();
393
394        let idx_a = std::fs::read(dest_a.path().join("index.bin")).unwrap();
395        let idx_b = std::fs::read(dest_b.path().join("index.bin")).unwrap();
396        assert_eq!(idx_a, idx_b, "index.bin must be byte-deterministic");
397
398        let pack_a = std::fs::read(dest_a.path().join("data.pack")).unwrap();
399        let pack_b = std::fs::read(dest_b.path().join("data.pack")).unwrap();
400        assert_eq!(pack_a, pack_b, "data.pack must be byte-deterministic");
401    }
402
403    #[test]
404    fn pack_round_trips_through_packloader() {
405        let proj = tempdir().unwrap();
406        let dest = tempdir().unwrap();
407        seed_project(proj.path());
408
409        let out = PackBuilder::new(proj.path(), dest.path()).build().unwrap();
410
411        // Read back the index, hand it to a VFS, and read each asset through
412        // a PackLoader to confirm the offsets/sizes line up with the bytes
413        // we appended (and that the leading 16-byte header is consumed
414        // transparently by PackLoader's seek-relative-to-asset-region logic).
415        let index_bytes = std::fs::read(&out.index_bin).unwrap();
416        let vfs = VirtualFileSystem::new(&index_bytes).unwrap();
417        let pack_file = std::fs::File::open(&out.data_pack).unwrap();
418        let pack_loader = PackLoader::new(pack_file).expect("valid pack header");
419        assert_eq!(pack_loader.header().asset_count, vfs.asset_count() as u32);
420        let metrics = Arc::new(MetricsRegistry::new());
421        let mut svc =
422            AssetService::new(&index_bytes, Box::new(pack_loader), metrics, None).unwrap();
423
424        // Verify each asset by raw load_raw, since we don't register any
425        // decoders here (binary blobs aren't real PNGs).
426        let known = [
427            ("scenes/default.kscene", &b"SCENE"[..]),
428            ("textures/a.png", &b"PNG-A"[..]),
429            ("textures/b.png", &b"PNG-B"[..]),
430        ];
431        for (rel, expected) in known {
432            let uuid = AssetUUID::new_v5(rel);
433            assert!(
434                vfs.get_metadata(&uuid).is_some(),
435                "uuid for {} missing",
436                rel
437            );
438            let bytes = svc.load_raw(&uuid).unwrap();
439            assert_eq!(bytes, expected, "round-trip mismatch for {}", rel);
440        }
441    }
442
443    #[test]
444    fn progress_channel_emits_started_and_finished() {
445        let proj = tempdir().unwrap();
446        let dest = tempdir().unwrap();
447        seed_project(proj.path());
448
449        let (tx, rx) = crossbeam_channel::unbounded::<PackProgress>();
450        let _ = PackBuilder::new(proj.path(), dest.path())
451            .with_progress(tx)
452            .build()
453            .unwrap();
454
455        let mut started = 0usize;
456        let mut finished = 0usize;
457        while let Ok(ev) = rx.try_recv() {
458            match ev {
459                PackProgress::Started { current, total, .. } => {
460                    assert_eq!(total, 3);
461                    assert!(current < total);
462                    started += 1;
463                }
464                PackProgress::Finished { asset_count, .. } => {
465                    assert_eq!(asset_count, 3);
466                    finished += 1;
467                }
468                PackProgress::Failed { .. } => panic!("unexpected Failed event"),
469            }
470        }
471        assert_eq!(started, 3);
472        assert_eq!(finished, 1);
473    }
474
475    #[test]
476    fn empty_assets_root_produces_empty_pack() {
477        let proj = tempdir().unwrap();
478        let dest = tempdir().unwrap();
479        let out = PackBuilder::new(proj.path(), dest.path()).build().unwrap();
480        assert_eq!(out.asset_count, 0);
481        assert_eq!(out.pack_bytes(), 0);
482        // Both files exist but are zero-byte.
483        assert!(out.data_pack.exists());
484        assert!(out.index_bin.exists());
485    }
486}
khora_io/asset/pack_builder.rs

khora_io/asset/
pack_builder.rs
