khora_io/asset/
pack.rs

1// Copyright 2025 eraflo
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Pack-based asset loader for release mode.
16//!
17//! `data.pack` files start with a 16-byte header so the loader can fail
18//! fast on the wrong file (e.g. the user accidentally renamed something to
19//! `data.pack`) and reject pack-format versions it wasn't built against.
20//! See [`PACK_HEADER_SIZE`] and [`PackHeader`] for the byte layout, and
21//! `crates/khora-io/src/asset/pack_builder.rs` for the writer side.
22
23use anyhow::{bail, Context, Result};
24use khora_core::asset::{AssetSource, CompressionKind};
25use std::{
26    fs::File,
27    io::{Read, Seek, SeekFrom, Write},
28};
29
30use super::AssetIo;
31
32/// 8-byte magic prefix that identifies a Khora pack archive on disk.
33///
34/// Trailing NUL keeps it printable in `od -c` / hex dumps. Persisted as
35/// raw bytes — endianness-independent.
36pub const PACK_MAGIC: &[u8; 8] = b"KHORAPK\0";
37
38/// Current pack format version. Bumped when the on-disk layout changes in
39/// a non-additive way; older runtimes refuse newer packs by reading the
40/// version from the header.
41///
42/// **v2** — adds 8 bytes to the header (`flags` + reserved) and
43/// per-entry compression (`uncompressed_size`, `compression`) in
44/// `AssetSource::Packed`. Older runtimes refuse v2 packs.
45pub const PACK_FORMAT_VERSION: u32 = 2;
46
47/// Total size in bytes of the leading [`PackHeader`] in `data.pack`.
48/// Asset offsets recorded in `index.bin` are **relative to the start of
49/// the asset region**, not to byte 0 — `PackLoader` adds this constant
50/// when seeking.
51pub const PACK_HEADER_SIZE: u64 = 24;
52
53/// Bit flag — at least one entry in this pack uses LZ4 compression.
54/// Diagnostics-only: per-entry compression is decided by
55/// `AssetSource::Packed.compression`. The flag is set when the writer
56/// applied LZ4 to any entry, so a quick header scan can tell whether the
57/// pack as a whole is mostly compressed.
58pub const PACK_FLAG_LZ4: u32 = 1 << 0;
59/// Bit flag — a `manifest.bin` BLAKE3 sidecar accompanies this pack.
60/// (Set by `PackBuilder` when integrity manifest emission is enabled.)
61pub const PACK_FLAG_MANIFEST: u32 = 1 << 1;
62
63/// Decoded form of the 24-byte header at the start of `data.pack`.
64///
65/// Byte layout (all little-endian):
66///
67/// ```text
68/// offset  size  field
69/// ──────  ────  ─────────────────────────────────────────────────────
70///      0     8  PACK_MAGIC (b"KHORAPK\0")
71///      8     4  format_version: u32  — must equal PACK_FORMAT_VERSION
72///     12     4  asset_count:    u32  — sanity check vs. index.bin
73///     16     4  flags:          u32  — bit 0 LZ4, bit 1 manifest
74///     20     4  reserved:       u32  — zero on writes, ignored on reads
75/// ```
76#[derive(Debug, Clone, Copy)]
77pub struct PackHeader {
78    /// Format version as written. Equals [`PACK_FORMAT_VERSION`] on a pack
79    /// produced by this engine; loader refuses anything else.
80    pub format_version: u32,
81    /// Number of assets the producer claims to have written into this
82    /// `data.pack`. Cross-checked against `index.bin.len()` at boot to
83    /// catch a mismatched pair (e.g. user shipped only one of the two
84    /// files).
85    pub asset_count: u32,
86    /// Pack-level feature flags (compression, manifest, …). See
87    /// `PACK_FLAG_*` constants.
88    pub flags: u32,
89}
90
91impl PackHeader {
92    /// Encodes the header into 24 raw bytes ready to be prepended to
93    /// `data.pack`. Used by `PackBuilder`.
94    pub fn to_bytes(&self) -> [u8; PACK_HEADER_SIZE as usize] {
95        let mut out = [0u8; PACK_HEADER_SIZE as usize];
96        out[0..8].copy_from_slice(PACK_MAGIC);
97        out[8..12].copy_from_slice(&self.format_version.to_le_bytes());
98        out[12..16].copy_from_slice(&self.asset_count.to_le_bytes());
99        out[16..20].copy_from_slice(&self.flags.to_le_bytes());
100        // Bytes 20..24 reserved — left zero.
101        out
102    }
103
104    /// Convenience for the writer: build the v2 header for a pack that
105    /// will contain `asset_count` blobs and optional feature flags.
106    pub fn v2(asset_count: u32, flags: u32) -> Self {
107        Self {
108            format_version: PACK_FORMAT_VERSION,
109            asset_count,
110            flags,
111        }
112    }
113}
114
115/// Pack-based asset loader for release mode.
116///
117/// Reads assets from a `.pack` archive file by seeking to the recorded
118/// offset (shifted by [`PACK_HEADER_SIZE`]) and reading the specified
119/// number of bytes.
120#[derive(Debug)]
121pub struct PackLoader {
122    pack_file: File,
123    header: PackHeader,
124}
125
126impl PackLoader {
127    /// Validates the leading header and returns a reader bound to
128    /// `pack_file`.
129    ///
130    /// Errors:
131    /// - file shorter than [`PACK_HEADER_SIZE`] or unreadable,
132    /// - magic doesn't match [`PACK_MAGIC`] (file isn't a Khora pack),
133    /// - `format_version` field doesn't match [`PACK_FORMAT_VERSION`]
134    ///   (this runtime can't read this pack — typically an older runtime
135    ///   reading a newer pack).
136    pub fn new(mut pack_file: File) -> Result<Self> {
137        let header =
138            read_and_validate_header(&mut pack_file).context("Pack header validation failed")?;
139        Ok(Self { pack_file, header })
140    }
141
142    /// Returns a reference to the parsed header. Useful for diagnostics
143    /// and for asserting `header.asset_count == vfs.asset_count()` at
144    /// startup.
145    pub fn header(&self) -> &PackHeader {
146        &self.header
147    }
148
149    /// Writes a fresh 24-byte header at the start of `out`. Convenience
150    /// helper for [`crate::asset::PackBuilder`] — keeps all the byte-
151    /// layout knowledge in one module.
152    pub fn write_header(out: &mut impl Write, asset_count: u32, flags: u32) -> Result<()> {
153        let header = PackHeader::v2(asset_count, flags);
154        out.write_all(&header.to_bytes())
155            .context("Failed to write pack header")
156    }
157}
158
159fn read_and_validate_header(file: &mut File) -> Result<PackHeader> {
160    file.seek(SeekFrom::Start(0))
161        .context("Failed to seek to pack file start")?;
162    let mut buf = [0u8; PACK_HEADER_SIZE as usize];
163    file.read_exact(&mut buf)
164        .context("Pack file is shorter than 24 bytes — not a Khora pack (or truncated)")?;
165
166    if &buf[0..8] != PACK_MAGIC {
167        bail!(
168            "Not a Khora pack archive (bad magic: expected {:?}, got {:?})",
169            std::str::from_utf8(PACK_MAGIC).unwrap_or("KHORAPK\\0"),
170            &buf[0..8]
171        );
172    }
173    // `buf` is exactly `PACK_HEADER_SIZE` bytes and `read_exact` succeeded, so
174    // each 4-byte window below is in bounds; propagate rather than unwrap so a
175    // future header-size change surfaces as an error instead of a panic.
176    let to_u32 = |slice: &[u8]| -> Result<u32> {
177        let arr: [u8; 4] = slice
178            .try_into()
179            .context("Pack header field is not 4 bytes — truncated or corrupt pack")?;
180        Ok(u32::from_le_bytes(arr))
181    };
182    let format_version = to_u32(&buf[8..12])?;
183    if format_version != PACK_FORMAT_VERSION {
184        bail!(
185            "Unsupported pack format version {} (this runtime supports v{})",
186            format_version,
187            PACK_FORMAT_VERSION
188        );
189    }
190    let asset_count = to_u32(&buf[12..16])?;
191    let flags = to_u32(&buf[16..20])?;
192    Ok(PackHeader {
193        format_version,
194        asset_count,
195        flags,
196    })
197}
198
199impl AssetIo for PackLoader {
200    fn load_bytes(&mut self, source: &AssetSource) -> Result<Vec<u8>> {
201        match source {
202            AssetSource::Packed {
203                offset,
204                size,
205                uncompressed_size,
206                compression,
207            } => {
208                let mut buffer = vec![0; *size as usize];
209                // Asset offsets in `index.bin` are relative to the start
210                // of the asset region (i.e. after the header). We add the
211                // header size here so the index never has to know about
212                // the on-disk header.
213                self.pack_file
214                    .seek(SeekFrom::Start(PACK_HEADER_SIZE + *offset))
215                    .context("Failed to seek to asset location in pack file")?;
216                self.pack_file
217                    .read_exact(&mut buffer)
218                    .context("Failed to read asset bytes from pack file")?;
219
220                match compression {
221                    CompressionKind::None => Ok(buffer),
222                    CompressionKind::Lz4 => {
223                        let decompressed =
224                            lz4_flex::block::decompress(&buffer, *uncompressed_size as usize)
225                                .map_err(|e| anyhow::anyhow!("LZ4 decompress failed: {}", e))?;
226                        if decompressed.len() != *uncompressed_size as usize {
227                            bail!(
228                                "Pack: LZ4-decompressed size {} != recorded uncompressed_size {}",
229                                decompressed.len(),
230                                uncompressed_size
231                            );
232                        }
233                        Ok(decompressed)
234                    }
235                }
236            }
237            AssetSource::Path(_) => {
238                bail!("PackLoader does not support Path sources")
239            }
240        }
241    }
242}
243
244#[cfg(test)]
245mod tests {
246    use super::*;
247    use std::io::Write;
248    use tempfile::tempdir;
249
250    fn write_pack(dir: &std::path::Path, bytes: &[u8]) -> File {
251        let path = dir.join("data.pack");
252        let mut f = File::create(&path).unwrap();
253        f.write_all(bytes).unwrap();
254        f.sync_all().unwrap();
255        File::open(&path).unwrap()
256    }
257
258    fn valid_header_bytes(asset_count: u32) -> [u8; PACK_HEADER_SIZE as usize] {
259        PackHeader::v2(asset_count, 0).to_bytes()
260    }
261
262    #[test]
263    fn rejects_file_with_wrong_magic() {
264        let dir = tempdir().unwrap();
265        let f = write_pack(
266            dir.path(),
267            b"not_a_pack......\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
268        );
269        let err = PackLoader::new(f).unwrap_err();
270        let msg = format!("{err:#}");
271        assert!(msg.contains("bad magic"), "got: {msg}");
272    }
273
274    #[test]
275    fn rejects_file_shorter_than_header() {
276        let dir = tempdir().unwrap();
277        let f = write_pack(dir.path(), b"KHORAPK"); // 7 bytes, less than header
278        let err = PackLoader::new(f).unwrap_err();
279        let msg = format!("{err:#}");
280        assert!(msg.contains("shorter than 24 bytes"), "got: {msg}");
281    }
282
283    #[test]
284    fn rejects_unsupported_version() {
285        let dir = tempdir().unwrap();
286        let mut bytes = Vec::new();
287        bytes.extend_from_slice(PACK_MAGIC); // 8
288        bytes.extend_from_slice(&999u32.to_le_bytes()); // version
289        bytes.extend_from_slice(&0u32.to_le_bytes()); // asset_count
290        bytes.extend_from_slice(&0u32.to_le_bytes()); // flags
291        bytes.extend_from_slice(&0u32.to_le_bytes()); // reserved
292        let f = write_pack(dir.path(), &bytes);
293        let err = PackLoader::new(f).unwrap_err();
294        let msg = format!("{err:#}");
295        assert!(
296            msg.contains("Unsupported pack format version 999"),
297            "got: {msg}"
298        );
299    }
300
301    #[test]
302    fn accepts_valid_header_and_exposes_asset_count() {
303        let dir = tempdir().unwrap();
304        let mut bytes = valid_header_bytes(42).to_vec();
305        bytes.extend_from_slice(b"\xAA\xBB\xCC"); // some payload
306        let f = write_pack(dir.path(), &bytes);
307        let loader = PackLoader::new(f).unwrap();
308        assert_eq!(loader.header().format_version, PACK_FORMAT_VERSION);
309        assert_eq!(loader.header().asset_count, 42);
310    }
311
312    #[test]
313    fn load_bytes_reads_relative_to_header() {
314        let dir = tempdir().unwrap();
315        let mut bytes = valid_header_bytes(1).to_vec();
316        bytes.extend_from_slice(b"PAYLOAD");
317        let f = write_pack(dir.path(), &bytes);
318        let mut loader = PackLoader::new(f).unwrap();
319        let got = loader
320            .load_bytes(&AssetSource::Packed {
321                offset: 0,
322                size: 7,
323                uncompressed_size: 7,
324                compression: CompressionKind::None,
325            })
326            .unwrap();
327        assert_eq!(got, b"PAYLOAD");
328    }
329
330    #[test]
331    fn load_bytes_decompresses_lz4() {
332        let dir = tempdir().unwrap();
333        let payload = b"PAYLOAD-ALL-COMPRESSIBLE-PAYLOAD-ALL-COMPRESSIBLE";
334        let compressed = lz4_flex::block::compress(payload);
335        let mut bytes = valid_header_bytes(1).to_vec();
336        bytes.extend_from_slice(&compressed);
337        let f = write_pack(dir.path(), &bytes);
338        let mut loader = PackLoader::new(f).unwrap();
339        let got = loader
340            .load_bytes(&AssetSource::Packed {
341                offset: 0,
342                size: compressed.len() as u64,
343                uncompressed_size: payload.len() as u64,
344                compression: CompressionKind::Lz4,
345            })
346            .unwrap();
347        assert_eq!(got, payload);
348    }
349}
khora_io/asset/pack.rs

khora_io/asset/
pack.rs
