lfg2025/archy
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 12 Wiki Activity

Stream purchased files into durable cache and avoid duplicate concurrent payments

Browse Source
This commit is contained in:
archipelago
2026-10-06 05:48:05 -04:00
parent df7677d23f
commit 9ce04627dd
9 changed files with 709 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/api/rpc/content.rs
+193 -110
View File
@@ -43,6 +43,22 @@ async fn reclaim_spent_ecash(data_dir: &std::path::Path, token: &str, backend: &
}
}
async fn bounded_seller_error(mut response: reqwest::Response) -> String {
let mut bytes = Vec::new();
let _ = tokio::time::timeout(std::time::Duration::from_secs(5), async {
while bytes.len() < 4096 {
match response.chunk().await {
Ok(Some(chunk)) => {
bytes.extend_from_slice(&chunk[..chunk.len().min(4096 - bytes.len())])
}
_ => break,
}
}
})
.await;
String::from_utf8_lossy(&bytes).into_owned()
}
/// Only pass through the peer's bounded, printable explanation; refund status
/// is always determined locally and must never come from the peer's wording.
fn seller_error_message(status: reqwest::StatusCode, body: &str) -> String {
@@ -80,6 +96,7 @@ async fn existing_paid_content(
onion: &str,
content_id: &str,
filename: Option<&str>,
cache_only: bool,
) -> Result<Option<serde_json::Value>> {
let owned = crate::content_owned::list_owned_checked(data_dir)
.await
@@ -93,35 +110,80 @@ async fn existing_paid_content(
}) else {
return Ok(None);
};
let (mime, bytes) = crate::content_owned::read_owned(data_dir, &item.onion, &item.content_id)
.await.context("This purchase is recorded, but its cached file is unavailable. No new payment was sent. Restore the cached file or contact the seller.")?;
let mut response = paid_content_response(&bytes, &mime, 0);
let mut response = cached_purchase_response(data_dir, &item.onion, &item.content_id, cache_only, 0).await
.context("This purchase is recorded, but its cached file is unavailable. No new payment was sent. Recover delivery without paying again.")?;
response["already_owned"] = serde_json::json!(true);
response["filename"] = serde_json::json!(item.filename);
Ok(Some(response))
}
// Updated clients open the persisted file through the Range-capable HTTP
// endpoint. Avoid putting two base64 copies of a large video in a JSON reply.
// Keep older clients compatible until both sides have upgraded.
fn invoice_download_response(bytes: &[u8], mime: &str, cache_only: bool) -> serde_json::Value {
if cache_only {
serde_json::json!({ "owned": true, "mime_type": mime, "size_bytes": bytes.len() })
} else {
paid_content_response(bytes, mime, 0)
async fn cached_purchase_response(
data_dir: &std::path::Path,
onion: &str,
content_id: &str,
cache_only: bool,
paid_sats: u64,
) -> Result<serde_json::Value> {
use tokio::io::AsyncReadExt;
let (mime, file) = crate::content_owned::open_owned(data_dir, onion, content_id)
.await?
.context("Purchased content is not cached")?;
let size = file.metadata().await?.len();
if cache_only || size > 16 * 1024 * 1024 {
return Ok(
serde_json::json!({"owned":true,"mime_type":mime,"size":size,"size_bytes":size,"paid_sats":paid_sats,"owned_content_id":content_id}),
);
}
let mut bytes = Vec::with_capacity(size as usize);
file.take(16 * 1024 * 1024 + 1)
.read_to_end(&mut bytes)
.await?;
anyhow::ensure!(
bytes.len() as u64 == size,
"Purchased file changed during reading"
);
let mut result = paid_content_response(&bytes, &mime, paid_sats);
result["owned_content_id"] = serde_json::json!(content_id);
Ok(result)
}
/// File purchases through an atomic no-clobber write in Files' own namespace.
async fn file_purchase_in_files(
async fn cache_peer_response(
data_dir: &std::path::Path,
onion: &str,
content_id: &str,
filename: &str,
mime: &str,
bytes: &[u8],
) -> Result<String> {
let folder = if mime.starts_with("image/") || mime.starts_with("video/") {
paid_sats: u64,
backend: &str,
response: reqwest::Response,
) -> Result<crate::content_owned::OwnedItem> {
let expected = response.content_length();
crate::content_owned::record_purchase_stream(
data_dir,
crate::content_owned::OwnedItem {
onion: onion.into(),
content_id: content_id.into(),
filename: filename.into(),
mime_type: mime.into(),
size_bytes: expected.unwrap_or(0),
paid_sats,
ecash_backend: backend.into(),
purchased_at: chrono::Utc::now().to_rfc3339(),
download_complete: false,
},
response.bytes_stream(),
expected,
)
.await
}
async fn file_cached_purchase_in_files(
data_dir: &std::path::Path,
item: &crate::content_owned::OwnedItem,
) -> Result<()> {
let folder = if item.mime_type.starts_with("image/") || item.mime_type.starts_with("video/") {
"Photos"
} else if mime.starts_with("audio/") {
} else if item.mime_type.starts_with("audio/") {
"Music"
} else {
"Documents"
@@ -131,19 +193,16 @@ async fn file_purchase_in_files(
tokio::fs::metadata(&root).await?.is_dir(),
"Files storage is unavailable"
);
let name = std::path::Path::new(filename)
let name = std::path::Path::new(&item.filename)
.file_name()
.and_then(|n| n.to_str())
.filter(|n| !n.is_empty())
.unwrap_or("download");
let path =
crate::container::filebrowser::save_new_file(&root.join(folder), name, bytes).await?;
Ok(format!(
"{folder}/{}",
path.file_name()
.and_then(|n| n.to_str())
.context("Invalid Files name")?
))
let (_, file) = crate::content_owned::open_owned(data_dir, &item.onion, &item.content_id)
.await?
.context("Purchase unavailable")?;
crate::container::filebrowser::save_new_file_from(&root.join(folder), name, file).await?;
Ok(())
}
impl RpcHandler {
@@ -540,6 +599,9 @@ impl RpcHandler {
return Err(anyhow::anyhow!("Invalid v3 onion address"));
}
crate::content_owned::validate_identity(onion, content_id)?;
let _purchase_lock = crate::content_owned::lock_seller_purchases(onion).await;
// NEVER pay twice for content we already own (2026-07-22: a file
// shared twice produced two catalog ids for the same bytes and the
// buyer paid both). Guard BEFORE any ecash is minted, matching both
@@ -551,6 +613,10 @@ impl RpcHandler {
onion,
content_id,
params.get("filename").and_then(|v| v.as_str()),
params
.get("cache_only")
.and_then(|v| v.as_bool())
.unwrap_or(false),
)
.await?
{
@@ -668,12 +734,11 @@ impl RpcHandler {
if response.status() == reqwest::StatusCode::PAYMENT_REQUIRED {
// A 402 can mean mint validation, network failure, underpayment,
// or an unaccepted mint. Do not invent a mint-mismatch diagnosis.
let body = response.text().await.unwrap_or_default();
drop(response);
tracing::warn!(
"paid download: seller {onion} rejected {used_backend} payment of {price_sats} sats: {body}"
"paid download: seller rejected {used_backend} payment of {price_sats} sats"
);
// Seller couldn't redeem the token — reclaim it so the buyer keeps
// their funds (the spent-but-unredeemed-notes case the user hit).
// Reclaim only proofs the mint still considers unspent.
let refund = reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
return Ok(serde_json::json!({
"error": format!("The seller could not verify the payment. {refund}")
@@ -682,8 +747,8 @@ impl RpcHandler {
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
tracing::warn!("paid download: seller {onion} returned {status}: {body}");
let body = bounded_seller_error(response).await;
tracing::warn!("paid download: seller {onion} returned {status}");
let refund = reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
return Ok(serde_json::json!({
"error": format!("{} {refund}", seller_error_message(status, &body))
@@ -700,59 +765,26 @@ impl RpcHandler {
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "application/octet-stream".to_string());
let bytes = match response.bytes().await {
Ok(bytes) => bytes,
Err(error) => {
tracing::warn!("paid download: response body failed: {error}");
let refund =
reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
return Ok(serde_json::json!({
"error": format!("The file transfer was interrupted after payment was sent. {refund}")
}));
}
};
// Persist the purchase so it "stays unlocked" for this buyer: cache the
// bytes + metadata keyed by (onion, content_id). The gallery then renders
// it unblurred and views it in-app from this cache — no re-payment and no
// reliance on a browser download (which silently fails on the mobile
// companion, the original "paid but never unlocked" report). Best-effort:
// a cache-write failure must not fail an already-paid download.
let filename = params
.get("filename")
.and_then(|v| v.as_str())
.unwrap_or(content_id)
.to_string();
let purchased_at = chrono::Utc::now().to_rfc3339();
if let Err(e) = crate::content_owned::record_purchase(
.unwrap_or(content_id);
let item = cache_peer_response(&self.config.data_dir,onion,content_id,filename,&mime_type,price_sats,used_backend,response)
.await.context("Paid file delivery could not be saved. Do not send another payment; recover this purchase first")?;
if let Err(error) = file_cached_purchase_in_files(&self.config.data_dir, &item).await {
tracing::warn!("Purchase cached; optional Files copy failed: {error:#}");
}
let mut result = cached_purchase_response(
&self.config.data_dir,
onion,
content_id,
&filename,
&mime_type,
&bytes,
params
.get("cache_only")
.and_then(|v| v.as_bool())
.unwrap_or(false),
price_sats,
used_backend,
&purchased_at,
)
.await
{
tracing::warn!("paid download: failed to cache purchased content (non-fatal): {e:#}");
}
// The durable purchased-content cache above is primary. A Files copy
// remains optional: a stopped FileBrowser must not undo a paid download.
let filed =
file_purchase_in_files(&self.config.data_dir, &filename, &mime_type, &bytes).await;
match filed {
Ok(path) => tracing::info!("paid download: filed into Files/{path}"),
Err(error) => tracing::warn!(
"paid download: optional Files copy failed; purchase cache retained: {error}"
),
}
tracing::info!("paid download: received {} bytes from {onion} (paid {price_sats} sats via {used_backend})", bytes.len());
let mut result = paid_content_response(&bytes, &mime_type, price_sats);
.await?;
result["ecash_backend"] = serde_json::json!(used_backend);
Ok(result)
}
@@ -900,14 +932,27 @@ impl RpcHandler {
return Err(anyhow::anyhow!("Invalid payment_hash"));
}
crate::content_owned::validate_identity(onion, content_id)?;
let _purchase_lock = crate::content_owned::lock_seller_purchases(onion).await;
let cache_only = params
.get("cache_only")
.and_then(|v| v.as_bool())
.unwrap_or(false);
if let Some((mime, bytes)) =
crate::content_owned::read_owned(&self.config.data_dir, onion, content_id).await
if crate::content_owned::list_owned_checked(&self.config.data_dir)
.await?
.iter()
.any(|item| {
item.onion == onion && item.content_id == content_id && item.download_complete
})
{
return Ok(invoice_download_response(&bytes, &mime, cache_only));
return cached_purchase_response(
&self.config.data_dir,
onion,
content_id,
cache_only,
0,
)
.await;
}
// Older sellers only mark settlement during status polling. Always
// perform that handshake before requesting bytes; retries never pay.
@@ -976,36 +1021,29 @@ impl RpcHandler {
.next()
.unwrap_or("application/octet-stream")
.to_string();
let bytes = response
.bytes()
.await
.context("Paid file transfer interrupted; retry the download without paying again")?;
let filename = params
.get("filename")
.and_then(|v| v.as_str())
.unwrap_or(content_id);
crate::content_owned::record_purchase(
let item = cache_peer_response(
&self.config.data_dir,
onion,
content_id,
filename,
&mime,
&bytes,
params
.get("price_sats")
.and_then(|v| v.as_u64())
.unwrap_or(0),
"lightning",
&chrono::Utc::now().to_rfc3339(),
response,
)
.await
.context("Paid file could not be saved; retry the download without paying again")?;
if let Err(error) =
file_purchase_in_files(&self.config.data_dir, filename, &mime, &bytes).await
{
.context("Paid file could not be saved; retry delivery without paying again")?;
if let Err(error) = file_cached_purchase_in_files(&self.config.data_dir, &item).await {
tracing::warn!("Lightning purchase cached; optional Files copy failed: {error:#}");
}
Ok(invoice_download_response(&bytes, &mime, cache_only))
cached_purchase_response(&self.config.data_dir, onion, content_id, cache_only, 0).await
}
/// Buyer side (#46): ask the seller for a fresh on-chain address to pay.
@@ -1462,20 +1500,18 @@ impl RpcHandler {
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow::anyhow!("Missing content_id"))?;
match crate::content_owned::read_owned(&self.config.data_dir, onion, content_id).await {
Some((mime_type, bytes)) => {
use base64::Engine;
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
Ok(serde_json::json!({
"data": encoded,
"size": bytes.len(),
"mime_type": mime_type,
}))
}
None => Ok(serde_json::json!({
"error": "You don't own this item yet, or its cached copy is missing."
})),
}
existing_paid_content(
&self.config.data_dir,
onion,
content_id,
params.get("filename").and_then(|v| v.as_str()),
params
.get("cache_only")
.and_then(|v| v.as_bool())
.unwrap_or(false),
)
.await?
.context("Purchased content is not cached")
}
}
@@ -1486,15 +1522,62 @@ mod tests;
#[cfg(test)]
mod invoice_delivery_response_tests {
use super::*;
#[test]
fn cached_delivery_avoids_base64_but_keeps_old_clients_compatible() {
let cached = invoice_download_response(b"paid bytes", "video/mp4", true);
#[tokio::test]
async fn cached_delivery_avoids_base64_and_legacy_small_reads_remain_compatible() {
let dir = tempfile::tempdir().unwrap();
crate::content_owned::record_purchase(
dir.path(),
"seller.onion",
"film",
"film",
"video/mp4",
b"paid bytes",
1,
"cashu",
"now",
)
.await
.unwrap();
let cached = cached_purchase_response(dir.path(), "seller.onion", "film", true, 0)
.await
.unwrap();
assert_eq!(cached["owned"], true);
assert_eq!(cached["size_bytes"], 10);
assert!(cached.get("data").is_none());
assert!(cached.get("data_base64").is_none());
let legacy = invoice_download_response(b"paid bytes", "video/mp4", false);
let alias = existing_paid_content(
dir.path(),
"seller.onion",
"new-catalog-id",
Some("film"),
true,
)
.await
.unwrap()
.unwrap();
assert_eq!(alias["owned_content_id"], "film");
let legacy = cached_purchase_response(dir.path(), "seller.onion", "film", false, 0)
.await
.unwrap();
assert_eq!(legacy["data"], "cGFpZCBieXRlcw==");
assert_eq!(legacy["data"], legacy["data_base64"]);
let mut entry = crate::content_owned::list_owned_checked(dir.path())
.await
.unwrap()
.remove(0);
entry.content_id = "incomplete".into();
let stream = futures_util::stream::iter([Ok::<_, std::io::Error>(
bytes::Bytes::from_static(b"part"),
)]);
assert!(
crate::content_owned::record_purchase_stream(dir.path(), entry, stream, Some(10))
.await
.is_err()
);
assert!(
existing_paid_content(dir.path(), "seller.onion", "incomplete", None, true)
.await
.is_err()
);
}
}
core/archipelago/src/container/filebrowser.rs
+97 -2
View File
@@ -212,6 +212,27 @@ pub async fn save_new_file(dir: &Path, name: &str, bytes: &[u8]) -> Result<PathB
save_new_file_with(dir, name, bytes, write_via_userns).await
}
/// Copy the already-owned file with bounded buffers, retaining the no-clobber
/// and Files namespace rules used by small purchases.
pub async fn save_new_file_from(dir: &Path, name: &str, mut source: fs::File) -> Result<PathBuf> {
use tokio::io::AsyncSeekExt;
validate_filename(name)?;
match fs::symlink_metadata(dir).await {
Ok(meta) => anyhow::ensure!(meta.is_dir(), "Files destination is not a directory"),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error.into()),
}
source.seek(std::io::SeekFrom::Start(0)).await?;
match write_direct_stream(dir, name, &mut source).await {
Ok(path) => Ok(path),
Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
source.seek(std::io::SeekFrom::Start(0)).await?;
write_via_userns_stream(dir.to_owned(), name.to_owned(), source).await
}
Err(error) => Err(error).context("Saving purchased file"),
}
}
fn validate_filename(name: &str) -> Result<()> {
anyhow::ensure!(
!name.is_empty()
@@ -291,8 +312,15 @@ impl Drop for PendingFile {
}
async fn write_direct(dir: &Path, name: &str, bytes: &[u8]) -> std::io::Result<PathBuf> {
write_direct_stream(dir, name, &mut &bytes[..]).await
}
async fn write_direct_stream<R: tokio::io::AsyncRead + Unpin>(
dir: &Path,
name: &str,
source: &mut R,
) -> std::io::Result<PathBuf> {
use std::os::unix::fs::PermissionsExt;
use tokio::io::AsyncWriteExt;
fs::create_dir_all(dir).await?;
let temp_path = dir.join(format!(".archy-saving-{}", uuid::Uuid::new_v4()));
let mut file = fs::OpenOptions::new()
@@ -302,7 +330,7 @@ async fn write_direct(dir: &Path, name: &str, bytes: &[u8]) -> std::io::Result<P
.open(&temp_path)
.await?;
let temp = PendingFile(temp_path);
file.write_all(bytes).await?;
tokio::io::copy(source, &mut file).await?;
file.set_permissions(std::fs::Permissions::from_mode(0o644))
.await?;
file.sync_all().await?;
@@ -397,10 +425,77 @@ async fn write_via_userns(dir: PathBuf, name: String, bytes: Vec<u8>) -> Result<
.context("Files namespace writer timed out")?
}
async fn write_via_userns_stream(
dir: PathBuf,
name: String,
mut source: fs::File,
) -> Result<PathBuf> {
let expected = source.metadata().await?.len();
let mut child = tokio::process::Command::new("podman")
.args(["unshare", "sh", "-c", WRITE_VIA_USERNS, "sh"])
.arg(&dir)
.arg(&name)
.arg(expected.to_string())
.kill_on_drop(true)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
.spawn()
.context("Starting Files namespace writer")?;
let mut stdin = child.stdin.take().context("Files writer stdin missing")?;
tokio::time::timeout(std::time::Duration::from_secs(900), async {
let count = tokio::io::copy(&mut source, &mut stdin).await?;
drop(stdin);
let output = child.wait_with_output().await?;
anyhow::ensure!(
count == expected && output.status.success(),
"Files copy failed; purchased cache is retained"
);
let chosen = String::from_utf8(output.stdout).context("Invalid Files response")?;
validate_filename(&chosen)?;
anyhow::ensure!(
(1..=100).any(|n| numbered_name(&name, n) == chosen),
"Unexpected Files destination"
);
Ok(dir.join(chosen))
})
.await
.context("Files namespace writer timed out")?
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn streamed_purchase_preserves_existing_file_and_exact_large_copy() {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("film.mp4"), b"keep")
.await
.unwrap();
let source = dir.path().join("source");
let bytes = vec![17; 2 * 1024 * 1024];
fs::write(&source, &bytes).await.unwrap();
let path = save_new_file_from(
dir.path(),
"film.mp4",
fs::File::open(&source).await.unwrap(),
)
.await
.unwrap();
assert_eq!(path.file_name().unwrap(), "film (2).mp4");
assert_eq!(fs::read(path).await.unwrap(), bytes);
assert_eq!(
fs::read(dir.path().join("film.mp4")).await.unwrap(),
b"keep"
);
assert!(!std::fs::read_dir(dir.path()).unwrap().any(|entry| entry
.unwrap()
.file_name()
.to_string_lossy()
.starts_with(".archy-saving")));
}
#[tokio::test]
async fn cloud_credentials_use_unique_record_and_never_default_password() {
let dir = tempfile::tempdir().unwrap();
core/archipelago/src/content_owned.rs
+288 -24
View File
@@ -16,6 +16,29 @@ use tokio::{fs, io::AsyncWriteExt, sync::Mutex};
static PURCHASE_WRITES: Mutex<()> = Mutex::const_new(());
// Serialize purchases per seller so duplicate UI requests cannot both pass the
// ownership check and spend. Weak entries are pruned between acquisitions.
pub async fn lock_seller_purchases(onion: &str) -> tokio::sync::OwnedMutexGuard<()> {
use std::sync::{Arc, OnceLock, Weak};
static LOCKS: OnceLock<std::sync::Mutex<std::collections::HashMap<String, Weak<Mutex<()>>>>> =
OnceLock::new();
let lock = {
let mut locks = LOCKS
.get_or_init(Default::default)
.lock()
.unwrap_or_else(|e| e.into_inner());
locks.retain(|_, value| value.strong_count() > 0);
if let Some(lock) = locks.get(onion).and_then(Weak::upgrade) {
lock
} else {
let lock = Arc::new(Mutex::new(()));
locks.insert(onion.into(), Arc::downgrade(&lock));
lock
}
};
lock.lock_owned().await
}
const OWNED_DIR: &str = "purchased-content";
const OWNED_INDEX: &str = "owned.json";
@@ -32,6 +55,13 @@ pub struct OwnedItem {
pub ecash_backend: String,
/// RFC3339 timestamp; best-effort, empty if the clock was unavailable.
pub purchased_at: String,
/// False means payment succeeded but delivery still needs recovery.
#[serde(default = "completed")]
pub download_complete: bool,
}
fn completed() -> bool {
true
}
#[derive(Debug, Default, Serialize, Deserialize)]
@@ -81,27 +111,49 @@ async fn load_index(data_dir: &Path) -> OwnedIndex {
load_index_checked(data_dir).await.unwrap_or_default()
}
struct PendingFile(PathBuf);
impl Drop for PendingFile {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
pub fn validate_identity(onion: &str, content_id: &str) -> Result<()> {
anyhow::ensure!(
!onion.is_empty()
&& !content_id.is_empty()
&& onion != "."
&& content_id != "."
&& !onion.contains("..")
&& !content_id.contains("..")
&& sanitize(onion) == onion
&& sanitize(content_id) == content_id,
"Invalid purchase path"
);
Ok(())
}
async fn atomic_write(path: &Path, bytes: &[u8]) -> Result<()> {
let parent = path.parent().context("Purchase path has no parent")?;
fs::create_dir_all(parent).await?;
let temp = parent.join(format!(".purchase-{}.tmp", uuid::Uuid::new_v4()));
let temp = PendingFile(parent.join(format!(".purchase-{}.tmp", uuid::Uuid::new_v4())));
let result = async {
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&temp)
.open(&temp.0)
.await?;
file.write_all(bytes).await?;
file.sync_all().await?;
drop(file);
fs::rename(&temp, path).await?;
fs::rename(&temp.0, path).await?;
fs::File::open(parent).await?.sync_all().await?;
Ok::<_, anyhow::Error>(())
}
.await;
if result.is_err() {
let _ = fs::remove_file(&temp).await;
let _ = fs::remove_file(&temp.0).await;
}
result
}
@@ -131,6 +183,7 @@ pub async fn record_purchase(
ecash_backend: &str,
purchased_at: &str,
) -> Result<()> {
validate_identity(onion, content_id)?;
// Read-modify-write must be one serialized transaction. Never replace a
// damaged index with an empty one, and never expose partially written bytes.
let _lock = PURCHASE_WRITES.lock().await;
@@ -149,6 +202,7 @@ pub async fn record_purchase(
paid_sats,
ecash_backend: ecash_backend.to_string(),
purchased_at: purchased_at.to_string(),
download_complete: true,
};
if let Some(existing) = index
.items
@@ -162,6 +216,110 @@ pub async fn record_purchase(
save_index(data_dir, &index).await
}
fn require_cache_space(file: &fs::File, additional: u64) -> Result<()> {
use std::os::fd::AsRawFd;
let mut value = std::mem::MaybeUninit::<libc::statvfs>::uninit();
if unsafe { libc::fstatvfs(file.as_raw_fd(), value.as_mut_ptr()) } != 0 {
return Err(std::io::Error::last_os_error()).context("Checking purchase storage");
}
let value = unsafe { value.assume_init() };
let available = (value.f_bavail as u64).saturating_mul(value.f_frsize as u64);
anyhow::ensure!(
additional
.checked_add(256 * 1024 * 1024)
.is_some_and(|n| n <= available),
"Insufficient purchase storage; payment remains recorded for recovery"
);
Ok(())
}
/// Save a peer response without holding its body in memory. A durable incomplete
/// ownership record precedes body consumption, so interrupted delivery cannot be
/// mistaken for permission to send another payment. Invoice retries may complete it.
pub async fn record_purchase_stream<S, E>(
data_dir: &Path,
mut entry: OwnedItem,
mut stream: S,
expected: Option<u64>,
) -> Result<OwnedItem>
where
S: futures_util::Stream<Item = std::result::Result<bytes::Bytes, E>> + Unpin,
E: std::error::Error + Send + Sync + 'static,
{
use futures_util::StreamExt;
validate_identity(&entry.onion, &entry.content_id)?;
let path = bytes_path(data_dir, &entry.onion, &entry.content_id);
let parent = path.parent().context("Purchase has no parent")?;
fs::create_dir_all(parent).await?;
let temporary = PendingFile(parent.join(format!(".purchase-{}.tmp", uuid::Uuid::new_v4())));
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&temporary.0)
.await?;
entry.download_complete = false;
entry.size_bytes = expected.unwrap_or(0);
{
let _lock = PURCHASE_WRITES.lock().await;
let mut index = load_index_checked(data_dir).await?;
if let Some(old) = index
.items
.iter_mut()
.find(|i| i.onion == entry.onion && i.content_id == entry.content_id)
{
anyhow::ensure!(
!old.download_complete,
"Purchase is already cached; use the owned copy"
);
*old = entry.clone();
} else {
index.items.push(entry.clone());
}
save_index(data_dir, &index).await?;
}
require_cache_space(&file, expected.unwrap_or(0))?;
let mut received = 0u64;
while let Some(chunk) = stream.next().await {
let chunk =
chunk.context("Purchased file transfer interrupted; no new payment should be sent")?;
received = received
.checked_add(chunk.len() as u64)
.context("Content size overflow")?;
anyhow::ensure!(
expected.is_none_or(|n| received <= n),
"Seller exceeded the declared content length"
);
require_cache_space(&file, chunk.len() as u64)?;
for part in chunk.chunks(64 * 1024) {
file.write_all(part).await?;
}
}
anyhow::ensure!(
expected.is_none_or(|n| received == n),
"Purchased file transfer ended early"
);
file.sync_all().await?;
drop(file);
let _lock = PURCHASE_WRITES.lock().await;
let mut index = load_index_checked(data_dir).await?;
fs::rename(&temporary.0, &path).await?;
fs::File::open(parent).await?.sync_all().await?;
entry.size_bytes = received;
entry.download_complete = true;
if let Some(old) = index
.items
.iter_mut()
.find(|i| i.onion == entry.onion && i.content_id == entry.content_id)
{
*old = entry.clone();
} else {
anyhow::bail!("Purchase ownership record disappeared; no new payment should be sent");
}
save_index(data_dir, &index).await?;
Ok(entry)
}
/// Payment decisions must not interpret an unreadable index as no purchases.
pub async fn list_owned_checked(data_dir: &Path) -> Result<Vec<OwnedItem>> {
Ok(load_index_checked(data_dir).await?.items)
@@ -197,17 +355,10 @@ pub async fn open_owned(
else {
return Ok(None);
};
// Reject path components even when called outside the HTTP route.
validate_identity(onion, content_id)?;
anyhow::ensure!(
!onion.is_empty()
&& !content_id.is_empty()
&& onion != "."
&& content_id != "."
&& !onion.contains("..")
&& !content_id.contains("..")
&& sanitize(onion) == onion
&& sanitize(content_id) == content_id,
"Invalid purchase path"
item.download_complete,
"Payment is recorded, but delivery is incomplete. Retry delivery without paying again."
);
let file = fs::OpenOptions::new()
.read(true)
@@ -228,22 +379,135 @@ pub async fn read_owned(
onion: &str,
content_id: &str,
) -> Option<(String, Vec<u8>)> {
let bytes = fs::read(bytes_path(data_dir, onion, content_id))
.await
.ok()?;
let mime = load_index(data_dir)
.await
.items
.into_iter()
.find(|i| i.onion == onion && i.content_id == content_id)
.map(|i| i.mime_type)
.unwrap_or_else(|| "application/octet-stream".to_string());
use tokio::io::AsyncReadExt;
let (mime, mut file) = open_owned(data_dir, onion, content_id).await.ok()??;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes).await.ok()?;
Some((mime, bytes))
}
#[cfg(test)]
mod tests {
use super::*;
fn entry() -> OwnedItem {
OwnedItem {
onion: "seller.onion".into(),
content_id: "film".into(),
filename: "film.mp4".into(),
mime_type: "video/mp4".into(),
size_bytes: 0,
paid_sats: 1,
ecash_backend: "lightning".into(),
purchased_at: "now".into(),
download_complete: false,
}
}
#[tokio::test]
async fn stream_records_incomplete_delivery_then_recovers_without_losing_ownership() {
let dir = tempfile::tempdir().unwrap();
let short = futures_util::stream::iter([Ok::<_, std::io::Error>(
bytes::Bytes::from_static(b"part"),
)]);
assert!(record_purchase_stream(dir.path(), entry(), short, Some(10))
.await
.is_err());
let pending = list_owned_checked(dir.path()).await.unwrap();
assert_eq!(pending.len(), 1);
assert!(!pending[0].download_complete);
assert!(open_owned(dir.path(), "seller.onion", "film")
.await
.is_err());
assert_eq!(
std::fs::read_dir(owned_root(dir.path()).join("seller.onion"))
.unwrap()
.count(),
0
);
let chunks = futures_util::stream::iter(
(0..64).map(|_| Ok::<_, std::io::Error>(bytes::Bytes::from(vec![42; 65536]))),
);
let item = record_purchase_stream(dir.path(), entry(), chunks, Some(4 * 1024 * 1024))
.await
.unwrap();
assert!(item.download_complete);
let (_, bytes) = read_owned(dir.path(), "seller.onion", "film")
.await
.unwrap();
assert_eq!(bytes.len(), 4 * 1024 * 1024);
assert!(bytes.iter().all(|b| *b == 42));
assert_eq!(list_owned_checked(dir.path()).await.unwrap().len(), 1);
}
#[tokio::test]
async fn cancelled_stream_cleans_partial_bytes_but_keeps_payment_record() {
let dir = tempfile::tempdir().unwrap();
let started = std::sync::Arc::new(tokio::sync::Notify::new());
let notify = started.clone();
let root = dir.path().to_path_buf();
let task = tokio::spawn(async move {
let stream = Box::pin(futures_util::stream::once(async move {
notify.notify_one();
std::future::pending::<std::result::Result<bytes::Bytes, std::io::Error>>().await
}));
record_purchase_stream(&root, entry(), stream, Some(10)).await
});
tokio::time::timeout(std::time::Duration::from_secs(10), started.notified())
.await
.unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
assert!(!list_owned_checked(dir.path()).await.unwrap()[0].download_complete);
assert_eq!(
std::fs::read_dir(owned_root(dir.path()).join("seller.onion"))
.unwrap()
.count(),
0
);
}
#[tokio::test]
async fn unsafe_purchase_paths_are_rejected_before_writing() {
let dir = tempfile::tempdir().unwrap();
for id in ["..", "../private", "/absolute", "a/b"] {
assert!(record_purchase(
dir.path(),
"seller.onion",
id,
"x",
"x",
b"x",
1,
"cashu",
"now"
)
.await
.is_err());
}
assert!(!owned_root(dir.path()).exists());
}
#[tokio::test]
async fn seller_purchase_lock_blocks_duplicates_but_not_another_seller() {
let first = lock_seller_purchases("one.onion").await;
assert!(tokio::time::timeout(
std::time::Duration::from_millis(10),
lock_seller_purchases("one.onion")
)
.await
.is_err());
let other = tokio::time::timeout(
std::time::Duration::from_secs(1),
lock_seller_purchases("two.onion"),
)
.await
.unwrap();
drop(first);
let _next = tokio::time::timeout(
std::time::Duration::from_secs(1),
lock_seller_purchases("one.onion"),
)
.await
.unwrap();
drop(other);
}
#[tokio::test]
async fn owned_stream_preserves_ownership_on_missing_corrupt_or_symlinked_bytes() {
let dir = tempfile::tempdir().unwrap();