microcosm.blue / microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
fork atom

Use record_id/subject tuple as return type for get_many_to_many

As
[discussed](https://tangled.org/microcosm.blue/microcosm-rs/pulls/7#comment-1528)
we replaced the existing nested structure of multiple record ids mapped
to one subject with a flat vector of record_id/subject tuples.

Compared to the proposed double-cursor we would need to impelement for
the latter, the former allows for simpler single-cursor and possible
confusion on the user side on how to use these.

authored by seoul.systems and committed by tangled.org b3146a05 62f7d7b4

+85 -93
unified split
-6
constellation/src/lib.rs
··· 50 50 } 51 51 } 52 52 53 - #[derive(Debug, PartialEq, Serialize, Deserialize, Clone, Default)] 54 - pub struct RecordsBySubject { 55 - pub subject: String, 56 - pub records: Vec<RecordId>, 57 - } 58 - 59 53 /// maybe the worst type in this repo, and there are some bad types 60 54 #[derive(Debug, Serialize, PartialEq)] 61 55 pub struct CountsByCount {
+17 -3
constellation/src/server/mod.rs
··· 18 18 use tokio_util::sync::CancellationToken; 19 19 20 20 use crate::storage::{LinkReader, Order, StorageStats}; 21 - use crate::{CountsByCount, Did, RecordId, RecordsBySubject}; 21 + use crate::{CountsByCount, Did, RecordId}; 22 22 23 23 mod acceptable; 24 24 mod filters; ··· 695 695 #[serde(default = "get_default_cursor_limit")] 696 696 limit: u64, 697 697 } 698 + #[derive(Debug, Serialize, Clone)] 699 + struct ManyToManyItem { 700 + link: RecordId, 701 + subject: String, 702 + } 698 703 #[derive(Template, Serialize)] 699 704 #[template(path = "get-many-to-many.html.j2")] 700 705 struct GetManyToManyItemsResponse { 701 - linking_records: Vec<RecordsBySubject>, 706 + items: Vec<ManyToManyItem>, 702 707 cursor: Option<OpaqueApiCursor>, 703 708 #[serde(skip_serializing)] 704 709 query: GetManyToManyItemsQuery, ··· 759 764 760 765 let cursor = paged.next.map(|next| ApiKeyedCursor { next }.into()); 761 766 767 + let items: Vec<ManyToManyItem> = paged 768 + .items 769 + .into_iter() 770 + .map(|(record_id, subject)| ManyToManyItem { 771 + link: record_id, 772 + subject, 773 + }) 774 + .collect(); 775 + 762 776 Ok(acceptable( 763 777 accept, 764 778 GetManyToManyItemsResponse { 765 - linking_records: paged.items, 779 + items, 766 780 cursor, 767 781 query: (*query).clone(), 768 782 },
+10 -8
constellation/src/storage/mem_store.rs
··· 1 1 use super::{ 2 2 LinkReader, LinkStorage, Order, PagedAppendingCollection, PagedOrderedCollection, StorageStats, 3 3 }; 4 - use crate::{ActionableEvent, CountsByCount, Did, RecordId, RecordsBySubject}; 4 + use crate::{ActionableEvent, CountsByCount, Did, RecordId}; 5 5 use anyhow::Result; 6 6 use links::CollectedLink; 7 7 use std::collections::{HashMap, HashSet}; ··· 244 244 after: Option<String>, 245 245 filter_dids: &HashSet<Did>, 246 246 filter_to_targets: &HashSet<String>, 247 - ) -> Result<PagedOrderedCollection<RecordsBySubject, String>> { 247 + ) -> Result<PagedOrderedCollection<(RecordId, String), String>> { 248 248 let empty_res = Ok(PagedOrderedCollection { 249 249 items: Vec::new(), 250 250 next: None, ··· 307 307 308 308 let mut items = grouped_links 309 309 .into_iter() 310 - .map(|(t, r)| RecordsBySubject { 311 - subject: t.0, 312 - records: r, 310 + .flat_map(|(target, records)| { 311 + records 312 + .iter() 313 + .map(move |r| (r.clone(), target.0.clone())) 314 + .collect::<Vec<_>>() 313 315 }) 314 316 .collect::<Vec<_>>(); 315 317 316 - items.sort_by(|a, b| a.subject.cmp(&b.subject)); 318 + items.sort_by(|a: &(RecordId, String), b| a.1.cmp(&b.1)); 317 319 318 320 items = items 319 321 .into_iter() 320 - .skip_while(|item| after.as_ref().map(|a| &item.subject <= a).unwrap_or(false)) 322 + .skip_while(|item| after.as_ref().map(|a| &item.1 <= a).unwrap_or(false)) 321 323 .take(limit as usize) 322 324 .collect(); 323 325 324 326 let next = if items.len() as u64 >= limit { 325 - items.last().map(|item| item.subject.clone()) 327 + items.last().map(|item| item.1.clone()) 326 328 } else { 327 329 None 328 330 };
+34 -44
constellation/src/storage/mod.rs
··· 1 - use crate::{ActionableEvent, CountsByCount, Did, RecordId, RecordsBySubject}; 1 + use crate::{ActionableEvent, CountsByCount, Did, RecordId}; 2 2 use anyhow::Result; 3 3 use serde::{Deserialize, Serialize}; 4 4 use std::collections::{HashMap, HashSet}; ··· 145 145 after: Option<String>, 146 146 filter_dids: &HashSet<Did>, 147 147 filter_to_targets: &HashSet<String>, 148 - ) -> Result<PagedOrderedCollection<RecordsBySubject, String>>; 148 + ) -> Result<PagedOrderedCollection<(RecordId, String), String>>; 149 149 150 150 fn get_all_counts( 151 151 &self, ··· 1740 1740 &HashSet::new(), 1741 1741 )?, 1742 1742 PagedOrderedCollection { 1743 - items: vec![RecordsBySubject { 1744 - subject: "b.com".to_string(), 1745 - records: vec![RecordId { 1743 + items: vec![( 1744 + RecordId { 1746 1745 did: "did:plc:asdf".into(), 1747 1746 collection: "app.t.c".into(), 1748 1747 rkey: "asdf".into(), 1749 - }] 1750 - }], 1748 + }, 1749 + "b.com".to_string(), 1750 + )], 1751 1751 next: None, 1752 1752 } 1753 1753 ); 1754 1754 }); 1755 1755 1756 - test_each_storage!(get_m2m_filters, |storage| { 1756 + test_each_storage!(get_m2m_no_filters, |storage| { 1757 1757 storage.push( 1758 1758 &ActionableEvent::CreateLinks { 1759 1759 record_id: RecordId { ··· 1835 1835 3, 1836 1836 )?; 1837 1837 1838 - // Test without filters - should get all records grouped by secondary target 1838 + // Test without filters - should get all records as flat items 1839 1839 let result = storage.get_many_to_many( 1840 1840 "a.com", 1841 1841 "app.t.c", ··· 1846 1846 &HashSet::new(), 1847 1847 &HashSet::new(), 1848 1848 )?; 1849 - assert_eq!(result.items.len(), 2); 1849 + assert_eq!(result.items.len(), 4); 1850 1850 assert_eq!(result.next, None); 1851 - // Find b.com group 1852 - let b_group = result 1851 + // Check b.com items 1852 + let b_items: Vec<_> = result 1853 1853 .items 1854 1854 .iter() 1855 - .find(|group| group.subject == "b.com") 1856 - .unwrap(); 1857 - assert_eq!(b_group.subject, "b.com"); 1858 - assert_eq!(b_group.records.len(), 2); 1859 - assert!(b_group 1860 - .records 1855 + .filter(|(_, subject)| subject == "b.com") 1856 + .collect(); 1857 + assert_eq!(b_items.len(), 2); 1858 + assert!(b_items 1861 1859 .iter() 1862 - .any(|r| r.did.0 == "did:plc:asdf" && r.rkey == "asdf")); 1863 - assert!(b_group 1864 - .records 1860 + .any(|(r, _)| r.did.0 == "did:plc:asdf" && r.rkey == "asdf")); 1861 + assert!(b_items 1865 1862 .iter() 1866 - .any(|r| r.did.0 == "did:plc:asdf" && r.rkey == "asdf2")); 1867 - // Find c.com group 1868 - let c_group = result 1863 + .any(|(r, _)| r.did.0 == "did:plc:asdf" && r.rkey == "asdf2")); 1864 + // Check c.com items 1865 + let c_items: Vec<_> = result 1869 1866 .items 1870 1867 .iter() 1871 - .find(|group| group.subject == "c.com") 1872 - .unwrap(); 1873 - assert_eq!(c_group.subject, "c.com"); 1874 - assert_eq!(c_group.records.len(), 2); 1875 - assert!(c_group 1876 - .records 1868 + .filter(|(_, subject)| subject == "c.com") 1869 + .collect(); 1870 + assert_eq!(c_items.len(), 2); 1871 + assert!(c_items 1877 1872 .iter() 1878 - .any(|r| r.did.0 == "did:plc:fdsa" && r.rkey == "fdsa")); 1879 - assert!(c_group 1880 - .records 1873 + .any(|(r, _)| r.did.0 == "did:plc:fdsa" && r.rkey == "fdsa")); 1874 + assert!(c_items 1881 1875 .iter() 1882 - .any(|r| r.did.0 == "did:plc:fdsa" && r.rkey == "fdsa2")); 1876 + .any(|(r, _)| r.did.0 == "did:plc:fdsa" && r.rkey == "fdsa2")); 1883 1877 1884 1878 // Test with DID filter - should only get records from did:plc:fdsa 1885 1879 let result = storage.get_many_to_many( ··· 1892 1886 &HashSet::from_iter([Did("did:plc:fdsa".to_string())]), 1893 1887 &HashSet::new(), 1894 1888 )?; 1895 - assert_eq!(result.items.len(), 1); 1896 - let group = &result.items[0]; 1897 - assert_eq!(group.subject, "c.com"); 1898 - assert_eq!(group.records.len(), 2); 1899 - assert!(group.records.iter().all(|r| r.did.0 == "did:plc:fdsa")); 1889 + assert_eq!(result.items.len(), 2); 1890 + assert!(result.items.iter().all(|(_, subject)| subject == "c.com")); 1891 + assert!(result.items.iter().all(|(r, _)| r.did.0 == "did:plc:fdsa")); 1900 1892 1901 1893 // Test with target filter - should only get records linking to b.com 1902 1894 let result = storage.get_many_to_many( ··· 1909 1901 &HashSet::new(), 1910 1902 &HashSet::from_iter(["b.com".to_string()]), 1911 1903 )?; 1912 - assert_eq!(result.items.len(), 1); 1913 - let group = &result.items[0]; 1914 - assert_eq!(group.subject, "b.com"); 1915 - assert_eq!(group.records.len(), 2); 1916 - assert!(group.records.iter().all(|r| r.did.0 == "did:plc:asdf")); 1904 + assert_eq!(result.items.len(), 2); 1905 + assert!(result.items.iter().all(|(_, subject)| subject == "b.com")); 1906 + assert!(result.items.iter().all(|(r, _)| r.did.0 == "did:plc:asdf")); 1917 1907 }); 1918 1908 }
+6 -7
constellation/src/storage/rocks_store.rs
··· 2 2 ActionableEvent, LinkReader, LinkStorage, Order, PagedAppendingCollection, 3 3 PagedOrderedCollection, StorageStats, 4 4 }; 5 - use crate::{CountsByCount, Did, RecordId, RecordsBySubject}; 5 + use crate::{CountsByCount, Did, RecordId}; 6 6 use anyhow::{bail, Result}; 7 7 use bincode::Options as BincodeOptions; 8 8 use links::CollectedLink; ··· 1132 1132 after: Option<String>, 1133 1133 filter_dids: &HashSet<Did>, 1134 1134 filter_to_targets: &HashSet<String>, 1135 - ) -> Result<PagedOrderedCollection<RecordsBySubject, String>> { 1135 + ) -> Result<PagedOrderedCollection<(RecordId, String), String>> { 1136 1136 let collection = Collection(collection.to_string()); 1137 1137 let path = RPath(path.to_string()); 1138 1138 ··· 1241 1241 } 1242 1242 } 1243 1243 1244 - let mut items: Vec<RecordsBySubject> = Vec::with_capacity(grouped_links.len()); 1244 + let mut items: Vec<(RecordId, String)> = Vec::with_capacity(grouped_links.len()); 1245 1245 for (fwd_target_id, records) in &grouped_links { 1246 1246 let Some(target_key) = self 1247 1247 .target_id_table ··· 1253 1253 1254 1254 let target_string = target_key.0 .0; 1255 1255 1256 - items.push(RecordsBySubject { 1257 - subject: target_string, 1258 - records: records.clone(), 1259 - }); 1256 + records 1257 + .iter() 1258 + .for_each(|r| items.push((r.clone(), target_string.clone()))); 1260 1259 } 1261 1260 1262 1261 let next = if grouped_links.len() as u64 >= limit {
+6 -8
constellation/templates/get-many-to-many.html.j2
··· 23 23 24 24 <h3>Many-to-many links, most recent first:</h3> 25 25 26 - {% for group in linking_records %} 27 - <h4>Target: <code>{{ group.subject }}</code> <small>(<a href="/links/all?target={{ group.subject|urlencode }}">view all links</a>)</small></h4> 28 - {% for record in group.records %} 29 - <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ record.did().0 }} 30 - <strong>Collection</strong>: {{ record.collection }} 31 - <strong>RKey</strong>: {{ record.rkey }} 32 - -> <a href="https://pdsls.dev/at://{{ record.did().0 }}/{{ record.collection }}/{{ record.rkey }}">browse record</a></pre> 33 - {% endfor %} 26 + {% for item in items %} 27 + <pre style="display: block; margin: 1em 2em" class="code"><strong>Subject</strong>: <a href="/links/all?target={{ item.subject|urlencode }}">{{ item.subject }}</a> 28 + <strong>DID</strong>: {{ item.link.did().0 }} 29 + <strong>Collection</strong>: {{ item.link.collection }} 30 + <strong>RKey</strong>: {{ item.link.rkey }} 31 + -> <a href="https://pdsls.dev/at://{{ item.link.did().0 }}/{{ item.link.collection }}/{{ item.link.rkey }}">browse record</a></pre> 34 32 {% endfor %} 35 33 36 34 {% if let Some(c) = cursor %}
+12 -17
lexicons/blue.microcosm/links/getManyToMany.json
··· 4 4 "defs": { 5 5 "main": { 6 6 "type": "query", 7 - "description": "Get records that link to a primary subject, grouped by the secondary subjects they also reference", 7 + "description": "Get records that link to a primary subject along with the secondary subjects they also reference", 8 8 "parameters": { 9 9 "type": "params", 10 10 "required": ["subject", "source", "pathToOther"], ··· 50 50 "encoding": "application/json", 51 51 "schema": { 52 52 "type": "object", 53 - "required": ["linking_records"], 53 + "required": ["items"], 54 54 "properties": { 55 - "linking_records": { 55 + "items": { 56 56 "type": "array", 57 57 "items": { 58 58 "type": "ref", 59 - "ref": "#recordsBySubject" 59 + "ref": "#item" 60 60 } 61 61 }, 62 62 "cursor": { 63 - "type": "string", 64 - "description": "pagination cursor" 63 + "type": "string" 65 64 } 66 65 } 67 66 } 68 67 } 69 68 }, 70 - "recordsBySubject": { 69 + "item": { 71 70 "type": "object", 72 - "required": ["subject", "records"], 71 + "required": ["link", "subject"], 73 72 "properties": { 73 + "link": { 74 + "type": "ref", 75 + "ref": "#linkRecord" 76 + }, 74 77 "subject": { 75 - "type": "string", 76 - "description": "the secondary subject that these records link to" 77 - }, 78 - "records": { 79 - "type": "array", 80 - "items": { 81 - "type": "ref", 82 - "ref": "#linkRecord" 83 - } 78 + "type": "string" 84 79 } 85 80 } 86 81 },