1mod layout;
4mod tilde;
5
6pub use tilde::expand_tilde;
7
8use async_trait::async_trait;
9use gonzalo_core::{
10 BlobEntry, BlobStore, ContentHash, CoreError, DEFAULT_ANCESTOR_CAP, DeletePlan, DeleteResult,
11 Identity, KeyPrefix, PurgePlan, PutPlan, PutResult, Record, RecordKey, Result, Revision, Store,
12 now_ms, plan_delete, plan_purge, plan_put, plan_put_raw, validate_ancestor_cap,
13};
14use rustix::fs::{FlockOperation, flock};
15use std::io::{self, Write};
16use std::path::{Path, PathBuf};
17use std::sync::atomic::{AtomicU64, Ordering};
18use tokio::io::AsyncWriteExt;
19
20type PutPlanner = fn(Option<&Record>, Record, Option<Revision>, i64, usize) -> PutPlan;
24
25pub struct FsStore {
27 root: PathBuf,
28 cap: usize,
31}
32
33impl FsStore {
34 pub fn new(root: impl Into<PathBuf>) -> Self {
35 Self {
36 root: root.into(),
37 cap: DEFAULT_ANCESTOR_CAP,
38 }
39 }
40
41 pub fn with_ancestor_cap(mut self, cap: usize) -> Result<Self> {
44 self.cap = validate_ancestor_cap(cap)?;
45 Ok(self)
46 }
47
48 async fn read_record(&self, key: &RecordKey) -> Result<Option<Record>> {
49 let path = layout::record_path(&self.root, key);
50 match tokio::fs::read(&path).await {
51 Ok(bytes) => {
52 let rec: Record =
53 serde_json::from_slice(&bytes).map_err(|e| CoreError::Serde(e.to_string()))?;
54 Ok(Some(rec))
55 }
56 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
57 Err(e) => Err(CoreError::Backend(e.to_string())),
58 }
59 }
60
61 async fn listed_live(&self, key: &RecordKey) -> bool {
68 match self.read_record(key).await {
69 Ok(Some(rec)) => !rec.is_tombstone(),
70 Ok(None) => false,
71 Err(_) => true,
72 }
73 }
74}
75
76#[async_trait]
77impl Store for FsStore {
78 async fn get(&self, key: &RecordKey) -> Result<Option<Record>> {
79 Ok(self
81 .read_record(key)
82 .await?
83 .filter(|rec| !rec.is_tombstone()))
84 }
85
86 async fn put(&self, record: Record, expected: Option<Revision>) -> Result<PutResult> {
87 let root = self.root.clone();
94 let cap = self.cap;
95 tokio::task::spawn_blocking(move || put_locked(&root, record, expected, cap, plan_put))
96 .await
97 .map_err(|e| CoreError::Backend(format!("put task panicked: {e}")))?
98 }
99
100 async fn put_raw(&self, record: Record, expected: Option<Revision>) -> Result<PutResult> {
101 let root = self.root.clone();
106 let cap = self.cap;
107 tokio::task::spawn_blocking(move || put_locked(&root, record, expected, cap, plan_put_raw))
108 .await
109 .map_err(|e| CoreError::Backend(format!("put_raw task panicked: {e}")))?
110 }
111
112 async fn list(&self, prefix: &KeyPrefix) -> Result<Vec<RecordKey>> {
113 let mut keys = Vec::new();
116 collect_keys(&self.root, prefix, &mut keys).await?;
117 let mut out = Vec::with_capacity(keys.len());
118 for key in keys {
119 if self.listed_live(&key).await {
120 out.push(key);
121 }
122 }
123 Ok(out)
124 }
125
126 async fn delete_as(
128 &self,
129 key: &RecordKey,
130 expected: Option<Revision>,
131 author: Option<Identity>,
132 ) -> Result<DeleteResult> {
133 let root = self.root.clone();
138 let key = key.clone();
139 let cap = self.cap;
140 tokio::task::spawn_blocking(move || delete_locked(&root, &key, expected, cap, author))
141 .await
142 .map_err(|e| CoreError::Backend(format!("delete task panicked: {e}")))?
143 }
144
145 async fn get_raw(&self, key: &RecordKey) -> Result<Option<Record>> {
146 self.read_record(key).await
148 }
149
150 async fn list_raw(&self, prefix: &KeyPrefix) -> Result<Vec<RecordKey>> {
151 let mut out = Vec::new();
154 collect_keys(&self.root, prefix, &mut out).await?;
155 Ok(out)
156 }
157
158 async fn purge(&self, key: &RecordKey, expected: Revision) -> Result<DeleteResult> {
159 let root = self.root.clone();
164 let key = key.clone();
165 tokio::task::spawn_blocking(move || purge_locked(&root, &key, &expected))
166 .await
167 .map_err(|e| CoreError::Backend(format!("purge task panicked: {e}")))?
168 }
169}
170
171static BLOB_TMP_NONCE: AtomicU64 = AtomicU64::new(0);
174
175#[async_trait]
176impl BlobStore for FsStore {
177 async fn put_blob(&self, content: &[u8]) -> Result<ContentHash> {
178 let hash = ContentHash::of(content);
179 let path = layout::blob_path(&self.root, &hash);
180
181 if tokio::fs::try_exists(&path)
184 .await
185 .map_err(|e| CoreError::Backend(e.to_string()))?
186 {
187 return Ok(hash);
188 }
189 if let Some(parent) = path.parent() {
190 tokio::fs::create_dir_all(parent)
191 .await
192 .map_err(|e| CoreError::Backend(e.to_string()))?;
193 }
194
195 let nonce = BLOB_TMP_NONCE.fetch_add(1, Ordering::Relaxed);
199 let tmp = path.with_extension(format!("tmp.{}.{nonce}", std::process::id()));
200 let mut f = tokio::fs::File::create(&tmp)
207 .await
208 .map_err(|e| CoreError::Backend(e.to_string()))?;
209 f.write_all(content)
210 .await
211 .map_err(|e| CoreError::Backend(e.to_string()))?;
212 f.sync_all()
213 .await
214 .map_err(|e| CoreError::Backend(e.to_string()))?;
215 drop(f);
216 tokio::fs::rename(&tmp, &path)
217 .await
218 .map_err(|e| CoreError::Backend(e.to_string()))?;
219 if let Some(parent) = path.parent() {
220 let parent = parent.to_path_buf();
221 tokio::task::spawn_blocking(move || fsync_dir(&parent))
222 .await
223 .map_err(|e| CoreError::Backend(format!("fsync task panicked: {e}")))?
224 .map_err(|e| CoreError::Backend(e.to_string()))?;
225 }
226 Ok(hash)
227 }
228
229 async fn get_blob(&self, hash: &ContentHash) -> Result<Option<Vec<u8>>> {
230 let path = layout::blob_path(&self.root, hash);
231 match tokio::fs::read(&path).await {
232 Ok(bytes) => Ok(Some(bytes)),
233 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
234 Err(e) => Err(CoreError::Backend(e.to_string())),
235 }
236 }
237
238 async fn list_blobs(&self) -> Result<Vec<BlobEntry>> {
239 let dir = layout::blobs_dir(&self.root);
240 let mut entries = match tokio::fs::read_dir(&dir).await {
241 Ok(rd) => rd,
242 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
244 Err(e) => return Err(CoreError::Backend(e.to_string())),
245 };
246 let mut out = Vec::new();
247 while let Some(entry) = entries
248 .next_entry()
249 .await
250 .map_err(|e| CoreError::Backend(e.to_string()))?
251 {
252 let name = entry.file_name().to_string_lossy().to_string();
253 if !is_blob_hash(&name) {
260 continue;
261 }
262 let md = entry.metadata().await;
263 if let Some(e) = blob_entry_from_dir_entry(&name, md)? {
264 out.push(e);
265 }
266 }
267 Ok(out)
268 }
269
270 async fn has_blob(&self, hash: &ContentHash) -> Result<bool> {
271 let path = layout::blob_path(&self.root, hash);
272 tokio::fs::try_exists(&path)
273 .await
274 .map_err(|e| CoreError::Backend(e.to_string()))
275 }
276
277 async fn delete_blob(&self, hash: &ContentHash) -> Result<()> {
278 let path = layout::blob_path(&self.root, hash);
279 match tokio::fs::remove_file(&path).await {
280 Ok(()) => Ok(()),
281 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
283 Err(e) => Err(CoreError::Backend(e.to_string())),
284 }
285 }
286}
287
288fn blob_entry_from_dir_entry(
295 name: &str,
296 metadata: std::io::Result<std::fs::Metadata>,
297) -> Result<Option<BlobEntry>> {
298 if !is_blob_hash(name) {
299 return Ok(None);
300 }
301 let md = match metadata {
302 Ok(md) => md,
303 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
304 Err(e) => return Err(CoreError::Backend(e.to_string())),
305 };
306 let modified = md
307 .modified()
308 .map_err(|e| CoreError::Backend(e.to_string()))?;
309 Ok(Some(BlobEntry::from_system_time(
310 ContentHash(name.to_string()),
311 modified,
312 )))
313}
314
315fn is_blob_hash(name: &str) -> bool {
318 name.len() == 64
319 && name
320 .bytes()
321 .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
322}
323
324fn lock_record(path: &Path) -> Result<std::fs::File> {
335 if let Some(parent) = path.parent() {
336 std::fs::create_dir_all(parent).map_err(|e| CoreError::Backend(e.to_string()))?;
337 }
338 let lock_path = path.with_extension("json.lock");
339 let lock = std::fs::OpenOptions::new()
340 .create(true)
341 .truncate(false)
342 .write(true)
343 .open(&lock_path)
344 .map_err(|e| CoreError::Backend(e.to_string()))?;
345 flock(&lock, FlockOperation::LockExclusive).map_err(|e| CoreError::Backend(e.to_string()))?;
346 Ok(lock)
347}
348
349fn read_current(path: &Path) -> Result<Option<Record>> {
352 match std::fs::read(path) {
353 Ok(bytes) => serde_json::from_slice::<Record>(&bytes)
354 .map(Some)
355 .map_err(|e| CoreError::Serde(e.to_string())),
356 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
357 Err(e) => Err(CoreError::Backend(e.to_string())),
358 }
359}
360
361fn write_durable(path: &Path, record: &Record) -> Result<()> {
364 let bytes = serde_json::to_vec_pretty(record).map_err(|e| CoreError::Serde(e.to_string()))?;
365 let tmp = path.with_extension("json.tmp");
372 let mut f = std::fs::File::create(&tmp).map_err(|e| CoreError::Backend(e.to_string()))?;
373 f.write_all(&bytes)
374 .map_err(|e| CoreError::Backend(e.to_string()))?;
375 f.sync_all()
376 .map_err(|e| CoreError::Backend(e.to_string()))?;
377 drop(f);
378 std::fs::rename(&tmp, path).map_err(|e| CoreError::Backend(e.to_string()))?;
379 if let Some(parent) = path.parent() {
380 fsync_dir(parent).map_err(|e| CoreError::Backend(e.to_string()))?;
381 }
382 Ok(())
383}
384
385fn remove_durable(path: &Path) -> Result<()> {
388 match std::fs::remove_file(path) {
389 Ok(()) => {}
390 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
392 Err(e) => return Err(CoreError::Backend(e.to_string())),
393 }
394 if let Some(parent) = path.parent() {
395 fsync_dir(parent).map_err(|e| CoreError::Backend(e.to_string()))?;
396 }
397 Ok(())
398}
399
400fn put_locked(
410 root: &Path,
411 record: Record,
412 expected: Option<Revision>,
413 cap: usize,
414 plan: PutPlanner,
415) -> Result<PutResult> {
416 let key = record.key.clone();
417 let path = layout::record_path(root, &key);
418 let _lock = lock_record(&path)?;
420
421 let current = read_current(&path)?;
424 match plan(current.as_ref(), record, expected, now_ms(), cap) {
427 PutPlan::Write(stored) => {
428 write_durable(&path, &stored)?;
429 Ok(PutResult::Committed(stored.revision))
430 }
431 PutPlan::Conflict(conflict) => Ok(PutResult::Conflict(conflict)),
432 PutPlan::NotFound => Err(CoreError::NotFound(key)),
435 PutPlan::Rejected(reason) => Err(CoreError::Invalid(reason.to_string())),
439 }
440}
441
442fn purge_locked(root: &Path, key: &RecordKey, expected: &Revision) -> Result<DeleteResult> {
447 let path = layout::record_path(root, key);
448 let _lock = lock_record(&path)?;
450
451 let current = read_current(&path)?;
454 match plan_purge(current.as_ref(), expected) {
455 PurgePlan::Remove => {
456 remove_durable(&path)?;
457 Ok(DeleteResult::Deleted)
458 }
459 PurgePlan::Noop => Ok(DeleteResult::Deleted),
460 PurgePlan::Conflict(conflict) => Ok(DeleteResult::Conflict(conflict)),
461 }
462}
463
464fn delete_locked(
475 root: &Path,
476 key: &RecordKey,
477 expected: Option<Revision>,
478 cap: usize,
479 author: Option<Identity>,
480) -> Result<DeleteResult> {
481 let path = layout::record_path(root, key);
482 let _lock = lock_record(&path)?;
484
485 let current = read_current(&path)?;
488 match plan_delete(current.as_ref(), expected, now_ms(), cap, author.as_ref()) {
489 DeletePlan::Write(tombstone) => {
490 write_durable(&path, &tombstone)?;
491 Ok(DeleteResult::Deleted)
492 }
493 DeletePlan::Noop => Ok(DeleteResult::Deleted),
494 DeletePlan::Conflict(conflict) => Ok(DeleteResult::Conflict(conflict)),
495 }
496}
497
498fn fsync_dir(path: &Path) -> io::Result<()> {
505 let dir = std::fs::File::open(path)?;
506 match dir.sync_all() {
507 Ok(()) => Ok(()),
508 Err(e) if e.kind() == io::ErrorKind::InvalidInput => Ok(()),
509 Err(e) => Err(e),
510 }
511}
512
513async fn collect_keys(
515 root: &std::path::Path,
516 prefix: &KeyPrefix,
517 out: &mut Vec<RecordKey>,
518) -> Result<()> {
519 let mut namespaces = match tokio::fs::read_dir(root).await {
520 Ok(rd) => rd,
521 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
522 Err(e) => return Err(CoreError::Backend(e.to_string())),
523 };
524 while let Some(ns) = namespaces
525 .next_entry()
526 .await
527 .map_err(|e| CoreError::Backend(e.to_string()))?
528 {
529 if ns.file_type().await.map(|ft| !ft.is_dir()).unwrap_or(true) {
530 continue;
531 }
532 let ns_name = ns.file_name().to_string_lossy().to_string();
533 let mut cols = tokio::fs::read_dir(ns.path())
534 .await
535 .map_err(|e| CoreError::Backend(e.to_string()))?;
536 while let Some(col) = cols
537 .next_entry()
538 .await
539 .map_err(|e| CoreError::Backend(e.to_string()))?
540 {
541 if col.file_type().await.map(|ft| !ft.is_dir()).unwrap_or(true) {
542 continue;
543 }
544 let col_name = col.file_name().to_string_lossy().to_string();
545 let mut files = tokio::fs::read_dir(col.path())
546 .await
547 .map_err(|e| CoreError::Backend(e.to_string()))?;
548 while let Some(f) = files
549 .next_entry()
550 .await
551 .map_err(|e| CoreError::Backend(e.to_string()))?
552 {
553 let fname = f.file_name().to_string_lossy().to_string();
554 if let Some(id) = fname.strip_suffix(".json") {
555 let key = RecordKey::new(
558 gonzalo_core::decode_segment(&ns_name),
559 gonzalo_core::decode_segment(&col_name),
560 gonzalo_core::decode_segment(id),
561 );
562 if prefix.matches(&key) {
563 out.push(key);
564 }
565 }
566 }
567 }
568 }
569 Ok(())
570}
571
572#[cfg(test)]
573mod listing_tests {
574 use super::*;
575 use std::io::{Error, ErrorKind};
576
577 fn hash_name() -> String {
578 "a".repeat(64)
579 }
580
581 #[test]
582 fn a_blob_unlinked_before_its_metadata_read_is_skipped() {
583 let r = blob_entry_from_dir_entry(&hash_name(), Err(Error::from(ErrorKind::NotFound)));
584 assert!(matches!(r, Ok(None)));
585 }
586
587 #[test]
588 fn a_name_that_is_not_a_blob_hash_yields_no_entry() {
589 let dir = tempfile::tempdir().unwrap();
590 let f = dir.path().join("x");
591 std::fs::write(&f, b"x").unwrap();
592 for name in [
593 "not-a-hash",
594 "tmp.partial",
595 &"A".repeat(64),
596 &"a".repeat(63),
597 ] {
598 let r = blob_entry_from_dir_entry(name, std::fs::metadata(&f));
599 assert!(matches!(r, Ok(None)), "{name} must not be listed");
600 }
601 }
602
603 #[test]
604 fn any_other_metadata_failure_is_a_real_error() {
605 let r =
606 blob_entry_from_dir_entry(&hash_name(), Err(Error::from(ErrorKind::PermissionDenied)));
607 assert!(r.is_err());
608 }
609
610 #[test]
611 fn a_readable_blob_yields_an_entry_with_a_real_timestamp() {
612 let dir = tempfile::tempdir().unwrap();
613 let f = dir.path().join("x");
614 std::fs::write(&f, b"x").unwrap();
615 let md = std::fs::metadata(&f);
616 let e = blob_entry_from_dir_entry(&hash_name(), md)
617 .unwrap()
618 .expect("an entry");
619 assert_eq!(e.hash, ContentHash(hash_name()));
620 assert_ne!(e.modified_unix_ms, 0);
621 }
622}