//! Node-signed byte indexes, created only by a scan matching the producer's //! original signed full SHA. Metadata alone never authorizes streamed bytes. use crate::identity::NodeIdentity; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ fs::{File, OpenOptions}, io::{Read, Seek, SeekFrom, Write}, os::unix::fs::OpenOptionsExt, path::Path, }; pub(crate) const CHUNK_BYTES: usize = 64 * 1024; const MAX_BYTES: u64 = 16 * 1024 * 1024 * 1024; #[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub(crate) struct Binding { pub content_id: String, pub receipt_sha256: String, pub full_sha256: String, pub size: u64, } impl Binding { fn validate(&self) -> Result<()> { anyhow::ensure!( self.content_id .strip_prefix("registered_") .and_then(|id| uuid::Uuid::parse_str(id).ok()) .map(|id| format!("registered_{id}") == self.content_id) .unwrap_or(false) && self.size > 0 && self.size <= MAX_BYTES, "Invalid byte index binding" ); for value in [&self.receipt_sha256, &self.full_sha256] { anyhow::ensure!( value.len() == 64 && value .bytes() .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), "Invalid byte index digest" ); } Ok(()) } pub fn index_bytes(&self) -> Result { self.validate()?; Ok(self.size.div_ceil(CHUNK_BYTES as u64) * 32) } } #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Header { version: u32, binding: Binding, chunks_sha256: String, signature: String, } fn preimage(binding: &Binding, chunks_sha256: &str) -> Result> { Ok(serde_json::to_vec(&( "archipelago-rental-chunk-index-v1", CHUNK_BYTES, &binding.content_id, &binding.receipt_sha256, &binding.full_sha256, binding.size, chunks_sha256, ))?) } pub(crate) struct Index { pub binding: Binding, chunks: Vec<[u8; 32]>, pub commitment: String, } #[cfg(test)] fn scan_counts() -> &'static std::sync::Mutex> { static COUNTS: std::sync::OnceLock>> = std::sync::OnceLock::new(); COUNTS.get_or_init(Default::default) } #[cfg(test)] pub(crate) fn scans(content_id: &str) -> usize { *scan_counts().lock().unwrap().get(content_id).unwrap_or(&0) } impl Index { /// Exactly one whole-file scan. Caller runs it as a bounded background job; /// progress/cancellation cannot create or alter a purchase lease. pub fn scan( file: &mut File, binding: Binding, mut progress: impl FnMut(u64) -> Result<()>, ) -> Result { let bytes = binding.index_bytes()?; #[cfg(test)] { *scan_counts() .lock() .unwrap() .entry(binding.content_id.clone()) .or_default() += 1; } anyhow::ensure!( file.metadata()?.is_file() && file.metadata()?.len() == binding.size, "Snapshot size changed" ); file.seek(SeekFrom::Start(0))?; let mut chunks = Vec::with_capacity(bytes as usize / 32); let mut full = Sha256::new(); let mut buffer = [0u8; CHUNK_BYTES]; let mut read = 0; while read < binding.size { progress(read)?; let count = (binding.size - read).min(CHUNK_BYTES as u64) as usize; file.read_exact(&mut buffer[..count])?; full.update(&buffer[..count]); chunks.push(Sha256::digest(&buffer[..count]).into()); read += count as u64; } anyhow::ensure!( file.metadata()?.len() == binding.size && hex::encode(full.finalize()) == binding.full_sha256, "Registered snapshot content changed" ); progress(read)?; let mut digest = Sha256::new(); for chunk in &chunks { digest.update(chunk); } let commitment = hex::encode(digest.finalize()); file.seek(SeekFrom::Start(0))?; Ok(Self { binding, chunks, commitment, }) } /// Persist under the registration's existing verification flock. Sync rename /// is the commit point; no mutation is queued after a canceled future drops. pub fn save(&self, path: &Path, identity: &NodeIdentity) -> Result<()> { let header = Header { version: 1, binding: self.binding.clone(), chunks_sha256: self.commitment.clone(), signature: identity.sign(&preimage(&self.binding, &self.commitment)?), }; let encoded = serde_json::to_vec(&header)?; anyhow::ensure!(encoded.len() <= 4096, "Byte index header too large"); let parent = path.parent().context("Byte index directory missing")?; let temporary = parent.join(format!(".chunk-index-{}.tmp", uuid::Uuid::new_v4())); let result = (|| { let mut file = OpenOptions::new() .create_new(true) .write(true) .mode(0o600) .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) .open(&temporary)?; file.write_all(&(encoded.len() as u32).to_be_bytes())?; file.write_all(&encoded)?; for chunk in &self.chunks { file.write_all(chunk)?; } file.sync_all()?; std::fs::rename(&temporary, path)?; File::open(parent)?.sync_all()?; Ok(()) })(); if result.is_err() { let _ = std::fs::remove_file(temporary); } result } /// A signed index survives restart without treating an editable metadata /// cache as proof. Every delivered chunk is independently checked below. pub fn load(path: &Path, binding: &Binding, identity: &NodeIdentity) -> Result> { let expected = binding.index_bytes()?; let mut file = match OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK) .open(path) { Ok(file) => file, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(error.into()), }; let metadata = file.metadata()?; anyhow::ensure!( metadata.is_file() && metadata.len() <= expected + 4100, "Invalid byte index file" ); let mut len = [0u8; 4]; file.read_exact(&mut len)?; let len = u32::from_be_bytes(len) as usize; anyhow::ensure!( len <= 4096 && metadata.len() == 4 + len as u64 + expected, "Invalid byte index length" ); let mut header = vec![0; len]; file.read_exact(&mut header)?; let header: Header = serde_json::from_slice(&header)?; anyhow::ensure!( header.version == 1 && &header.binding == binding && NodeIdentity::verify( &identity.pubkey_hex(), &preimage(binding, &header.chunks_sha256)?, &header.signature )?, "Byte index signature or immutable binding changed" ); let mut chunks = Vec::with_capacity(expected as usize / 32); let mut digest = Sha256::new(); for _ in 0..expected / 32 { let mut chunk = [0; 32]; file.read_exact(&mut chunk)?; digest.update(chunk); chunks.push(chunk); } anyhow::ensure!( hex::encode(digest.finalize()) == header.chunks_sha256, "Byte index was altered" ); Ok(Some(Self { binding: binding.clone(), chunks, commitment: header.chunks_sha256, })) } /// Return only a slice of a completely verified aligned chunk. This bounds /// seek/range work to64KiB and detects same-inode/same-stamp byte changes. pub fn read_slice(&self, file: &mut File, start: u64, requested: usize) -> Result> { anyhow::ensure!( start < self.binding.size && requested > 0 && file.metadata()?.len() == self.binding.size, "Snapshot range or size changed" ); let chunk = start / CHUNK_BYTES as u64; let aligned = chunk * CHUNK_BYTES as u64; let size = (self.binding.size - aligned).min(CHUNK_BYTES as u64) as usize; let mut bytes = vec![0; size]; file.seek(SeekFrom::Start(aligned))?; file.read_exact(&mut bytes)?; let actual: [u8; 32] = Sha256::digest(&bytes).into(); anyhow::ensure!( self.chunks.get(chunk as usize) == Some(&actual), "Registered snapshot chunk changed; recover original entitlement" ); let offset = (start - aligned) as usize; Ok(bytes[offset..offset + requested.min(size - offset)].to_vec()) } } #[cfg(test)] mod tests { use super::*; fn binding(bytes: &[u8]) -> Binding { Binding { content_id: format!("registered_{}", uuid::Uuid::new_v4()), receipt_sha256: "ab".repeat(32), full_sha256: hex::encode(Sha256::digest(bytes)), size: bytes.len() as u64, } } #[tokio::test] async fn signed_index_reopens_without_whole_scan_and_seek_slices_reject_mutation() { let root = tempfile::tempdir().unwrap(); let identity = NodeIdentity::load_or_create(&root.path().join("identity")) .await .unwrap(); let bytes: Vec = (0..CHUNK_BYTES * 5 + 19).map(|n| (n % 251) as u8).collect(); let media = root.path().join("media"); std::fs::write(&media, &bytes).unwrap(); let mut file = File::open(&media).unwrap(); let mut progress = Vec::new(); let original = binding(&bytes); let index = Index::scan(&mut file, original.clone(), |n| { progress.push(n); Ok(()) }) .unwrap(); assert_eq!(progress.last(), Some(&(bytes.len() as u64))); index.save(&root.path().join("index"), &identity).unwrap(); drop(index); let loaded = Index::load(&root.path().join("index"), &original, &identity) .unwrap() .unwrap(); for start in [0, CHUNK_BYTES + 7, CHUNK_BYTES * 4, bytes.len() - 1] { let got = loaded.read_slice(&mut file, start as u64, 37).unwrap(); assert_eq!(got, bytes[start..start + got.len()]); } let mut corrupt = bytes.clone(); corrupt[CHUNK_BYTES + 9] ^= 1; std::fs::write(&media, corrupt).unwrap(); assert!(loaded .read_slice(&mut file, (CHUNK_BYTES + 7) as u64, 1) .is_err()); assert_eq!(loaded.read_slice(&mut file, 0, 5).unwrap(), &bytes[..5]); let mut saved = std::fs::read(root.path().join("index")).unwrap(); *saved.last_mut().unwrap() ^= 1; std::fs::write(root.path().join("index"), saved).unwrap(); assert!(Index::load(&root.path().join("index"), &original, &identity).is_err()); } #[test] fn canceled_or_wrong_hash_scan_never_produces_index() { let root = tempfile::tempdir().unwrap(); let bytes = vec![7; CHUNK_BYTES * 2]; let media = root.path().join("media"); std::fs::write(&media, &bytes).unwrap(); let mut file = File::open(media).unwrap(); assert!(Index::scan(&mut file, binding(&bytes), |n| { anyhow::ensure!(n == 0, "canceled"); Ok(()) }) .is_err()); let mut wrong = binding(&bytes); wrong.full_sha256 = "cd".repeat(32); assert!(Index::scan(&mut file, wrong, |_| Ok(())).is_err()); } }