tranquil.farm / tranquil-pds
Our Personal Data Server from scratch! tranquil.farm
oauth atproto pds rust postgresql objectstorage fun
fork atom

Sharded filesystem subdirs

lewis.moe e889aba9 8315d691

+88 -1
unified split
+88 -1
crates/tranquil-storage/src/lib.rs
··· 19 19 20 20 const MIN_PART_SIZE: usize = 5 * 1024 * 1024; 21 21 const EXDEV: i32 = 18; 22 + const CID_HASH_OFFSET: usize = 7; 23 + const SHARD_LENGTH: usize = 2; 24 + 25 + fn extract_cid_shard(key: &str) -> Option<&str> { 26 + let min_len = CID_HASH_OFFSET + SHARD_LENGTH; 27 + let is_cid = key.get(..3).map_or(false, |p| p.eq_ignore_ascii_case("baf")); 28 + (key.len() >= min_len && is_cid).then(|| &key[CID_HASH_OFFSET..CID_HASH_OFFSET + SHARD_LENGTH]) 29 + } 22 30 23 31 fn validate_key(key: &str) -> Result<(), StorageError> { 24 32 let dominated_by_traversal = key ··· 483 491 484 492 fn resolve_path(&self, key: &str) -> Result<PathBuf, StorageError> { 485 493 validate_key(key)?; 486 - Ok(self.base_path.join(key)) 494 + Ok(extract_cid_shard(key).map_or_else( 495 + || self.base_path.join(key), 496 + |shard| self.base_path.join(shard).join(key), 497 + )) 487 498 } 488 499 489 500 async fn atomic_write(&self, path: &Path, data: &[u8]) -> Result<(), StorageError> { ··· 751 762 } 752 763 753 764 impl<T> Pipe for T {} 765 + 766 + #[cfg(test)] 767 + mod tests { 768 + use super::*; 769 + 770 + #[test] 771 + fn extract_shard_from_raw_blob_cid() { 772 + let cid = "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"; 773 + assert_eq!(extract_cid_shard(cid), Some("hd")); 774 + } 775 + 776 + #[test] 777 + fn extract_shard_from_dag_cbor_cid() { 778 + let cid = "bafyreigdmqpykrgxyaxtlafqpqhzrb7qy2rh75nldvfd4tucqmqqme5yje"; 779 + assert_eq!(extract_cid_shard(cid), Some("gd")); 780 + } 781 + 782 + #[test] 783 + fn no_shard_for_temp_keys() { 784 + assert_eq!(extract_cid_shard("temp/abc123"), None); 785 + } 786 + 787 + #[test] 788 + fn no_shard_for_short_keys() { 789 + assert_eq!(extract_cid_shard("bafkrei"), None); 790 + assert_eq!(extract_cid_shard("baf"), None); 791 + assert_eq!(extract_cid_shard("ba"), None); 792 + assert_eq!(extract_cid_shard(""), None); 793 + } 794 + 795 + #[test] 796 + fn no_shard_for_non_cid_keys() { 797 + assert_eq!(extract_cid_shard("something/else/entirely"), None); 798 + assert_eq!(extract_cid_shard("Qmabcdefghijklmnop"), None); 799 + } 800 + 801 + #[test] 802 + fn extract_shard_case_insensitive() { 803 + let upper = "BAFKREIHDWDCEFGH4DQKJV67UZCMW7OJEE6XEDZDETOJUZJEVTENXQUVYKU"; 804 + let mixed = "BaFkReIhDwDcEfGh4DqKjV67UzCmW7OjEe6XeDzDeTojUzJevTeNxQuVyKu"; 805 + assert_eq!(extract_cid_shard(upper), Some("HD")); 806 + assert_eq!(extract_cid_shard(mixed), Some("hD")); 807 + } 808 + 809 + #[test] 810 + fn shard_at_minimum_length() { 811 + let cid = "bafkreiab"; 812 + assert_eq!(extract_cid_shard(cid), Some("ab")); 813 + } 814 + 815 + #[test] 816 + fn resolve_path_shards_cid_keys() { 817 + let base = PathBuf::from("/blobs"); 818 + let cid = "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"; 819 + 820 + let expected = PathBuf::from("/blobs/hd/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"); 821 + let result = extract_cid_shard(cid).map_or_else( 822 + || base.join(cid), 823 + |shard| base.join(shard).join(cid), 824 + ); 825 + assert_eq!(result, expected); 826 + } 827 + 828 + #[test] 829 + fn resolve_path_no_shard_for_temp() { 830 + let base = PathBuf::from("/blobs"); 831 + let key = "temp/abc123"; 832 + 833 + let expected = PathBuf::from("/blobs/temp/abc123"); 834 + let result = extract_cid_shard(key).map_or_else( 835 + || base.join(key), 836 + |shard| base.join(shard).join(key), 837 + ); 838 + assert_eq!(result, expected); 839 + } 840 + }