Dynamic post loading (#2) · witchcraft.systems/pds-dash@cff9eed

+12 -9

config.ts

··· 9 9 static readonly PDS_URL: string = "https://pds.witchcraft.systems"; 10 10 11 11 /** 12 - * The base URL of the frontend service for linking to replies 12 + * The base URL of the frontend service for linking to replies/quotes/accounts etc. 13 13 * @default "https://deer.social" 14 14 */ 15 15 static readonly FRONTEND_URL: string = "https://deer.social"; 16 16 17 17 /** 18 - * Maximum number of posts to show in the feed (across all users) 19 - * @default 100 18 + * Maximum number of posts to fetch from the PDS per request 19 + * Should be around 20 for about 10 users on the pds 20 + * The more users you have, the lower the number should be 21 + * since sorting is slow and is done on the frontend 22 + * @default 20 20 23 */ 21 - static readonly MAX_POSTS: number = 100; 24 + static readonly MAX_POSTS: number = 20; 22 25 23 26 /** 24 27 * Footer text for the dashboard ··· 27 30 static readonly FOOTER_TEXT: string = 28 31 "Astrally projected from <a href='https://witchcraft.systems' target='_blank'>witchcraft.systems</a>"; 29 32 30 - /** 31 - * Whether to show the posts that are in the future 32 - * @default false 33 - */ 34 - static readonly SHOW_FUTURE_POSTS: boolean = false; 33 + /** 34 + * Whether to show the posts that are in the future 35 + * @default false 36 + */ 37 + static readonly SHOW_FUTURE_POSTS: boolean = false; 35 38 }

+5

deno.lock

··· 8 8 "npm:@tsconfig/svelte@^5.0.4": "5.0.4", 9 9 "npm:moment@^2.30.1": "2.30.1", 10 10 "npm:svelte-check@^4.1.5": "4.1.6_svelte@5.28.1__acorn@8.14.1_typescript@5.7.3", 11 + "npm:svelte-infinite-loading@^1.4.0": "1.4.0", 11 12 "npm:svelte@^5.23.1": "5.28.1_acorn@8.14.1", 12 13 "npm:typescript@~5.7.2": "5.7.3", 13 14 "npm:vite@^6.3.1": "6.3.2_picomatch@4.0.2" ··· 415 416 "typescript" 416 417 ] 417 418 }, 419 + "svelte-infinite-loading@1.4.0": { 420 + "integrity": "sha512-Jo+f/yr/HmZQuIiiKKzAHVFXdAUWHW2RBbrcQTil8JVk1sCm/riy7KTJVzjBgQvHasrFQYKF84zvtc9/Y4lFYg==" 421 + }, 418 422 "svelte@5.28.1_acorn@8.14.1": { 419 423 "integrity": "sha512-iOa9WmfNG95lSOSJdMhdjJ4Afok7IRAQYXpbnxhd5EINnXseG0GVa9j6WPght4eX78XfFez45Fi+uRglGKPV/Q==", 420 424 "dependencies": [ ··· 476 480 "npm:@tsconfig/svelte@^5.0.4", 477 481 "npm:moment@^2.30.1", 478 482 "npm:svelte-check@^4.1.5", 483 + "npm:svelte-infinite-loading@^1.4.0", 479 484 "npm:svelte@^5.23.1", 480 485 "npm:typescript@~5.7.2", 481 486 "npm:vite@^6.3.1"

+2 -1

package.json

··· 13 13 "@atcute/bluesky": "^2.0.2", 14 14 "@atcute/client": "^3.0.1", 15 15 "@atcute/identity-resolver": "^0.1.2", 16 - "moment": "^2.30.1" 16 + "moment": "^2.30.1", 17 + "svelte-infinite-loading": "^1.4.0" 17 18 }, 18 19 "devDependencies": { 19 20 "@sveltejs/vite-plugin-svelte": "^5.0.3",

+33 -13

src/App.svelte

··· 1 1 <script lang="ts"> 2 2 import PostComponent from "./lib/PostComponent.svelte"; 3 3 import AccountComponent from "./lib/AccountComponent.svelte"; 4 - import { fetchAllPosts, Post, getAllMetadataFromPds } from "./lib/pdsfetch"; 4 + import InfiniteLoading from "svelte-infinite-loading"; 5 + import { getNextPosts, Post, getAllMetadataFromPds } from "./lib/pdsfetch"; 5 6 import { Config } from "../config"; 6 - const postsPromise = fetchAllPosts(); 7 7 const accountsPromise = getAllMetadataFromPds(); 8 + import { onMount } from "svelte"; 9 + 10 + let posts: Post[] = []; 11 + 12 + onMount(() => { 13 + // Fetch initial posts 14 + getNextPosts().then((initialPosts) => { 15 + posts = initialPosts; 16 + }); 17 + }); 18 + // Infinite loading function 19 + const onInfinite = ({ detail: { loaded, complete } } : { detail : { loaded : () => void, complete : () => void}}) => { 20 + getNextPosts().then((newPosts) => { 21 + if (newPosts.length > 0) { 22 + posts = [...posts, ...newPosts]; 23 + loaded(); 24 + } else { 25 + complete(); 26 + } 27 + }); 28 + }; 8 29 </script> 9 30 10 31 <main> ··· 26 47 <p>Error: {error.message}</p> 27 48 {/await} 28 49 29 - {#await postsPromise} 30 - <p>Loading...</p> 31 - {:then postsData} 32 - <div id="Feed"> 33 - <div id="spacer"></div> 34 - {#each postsData as postObject} 35 - <PostComponent post={postObject as Post} /> 36 - {/each} 37 - <div id="spacer"></div> 38 - </div> 39 - {/await} 50 + <div id="Feed"> 51 + <div id="spacer"></div> 52 + {#each posts as postObject} 53 + <PostComponent post={postObject as Post} /> 54 + {/each} 55 + <InfiniteLoading on:infinite={onInfinite} 56 + distance={0} 57 + /> 58 + <div id="spacer"></div> 59 + </div> 40 60 </div> 41 61 </main> 42 62

+2 -2

src/lib/PostComponent.svelte

··· 113 113 <div id="carouselControls"> 114 114 <button 115 115 id="prevBtn" 116 - on:click={prevImage} 116 + onclick={prevImage} 117 117 disabled={currentImageIndex === 0}>←</button 118 118 > 119 119 <div id="carouselIndicators"> ··· 125 125 </div> 126 126 <button 127 127 id="nextBtn" 128 - on:click={nextImage} 128 + onclick={nextImage} 129 129 disabled={currentImageIndex === post.imagesCid.length - 1} 130 130 >→</button 131 131 >

+149 -64

src/lib/pdsfetch.ts

··· 18 18 // import { AppBskyActorDefs } from "@atcute/client/lexicons"; 19 19 20 20 interface AccountMetadata { 21 - did: string; 21 + did: At.Did; 22 22 displayName: string; 23 23 handle: string; 24 24 avatarCid: string | null; 25 + currentCursor?: string; 25 26 } 27 + 28 + let accountsMetadata: AccountMetadata[] = []; 29 + 26 30 interface atUriObject { 27 31 repo: string; 28 32 collection: string; ··· 45 49 46 50 constructor( 47 51 record: ComAtprotoRepoListRecords.Record, 48 - account: AccountMetadata 52 + account: AccountMetadata, 49 53 ) { 50 54 this.postCid = record.cid; 51 55 this.recordName = processAtUri(record.uri).rkey; ··· 68 72 switch (post.embed?.$type) { 69 73 case "app.bsky.embed.images": 70 74 this.imagesCid = post.embed.images.map( 71 - (imageRecord: any) => imageRecord.image.ref.$link 75 + (imageRecord: any) => imageRecord.image.ref.$link, 72 76 ); 73 77 break; 74 78 case "app.bsky.embed.video": ··· 82 86 switch (post.embed.media.$type) { 83 87 case "app.bsky.embed.images": 84 88 this.imagesCid = post.embed.media.images.map( 85 - (imageRecord) => imageRecord.image.ref.$link 89 + (imageRecord) => imageRecord.image.ref.$link, 86 90 ); 87 91 88 92 break; ··· 118 122 return data.repos.map((repo: any) => repo.did) as At.Did[]; 119 123 }; 120 124 const getAccountMetadata = async ( 121 - did: `did:${string}:${string}` 122 - ): Promise<AccountMetadata> => { 125 + did: `did:${string}:${string}`, 126 + ) => { 123 127 // gonna assume self exists in the app.bsky.actor.profile 124 128 try { 125 129 const { data } = await rpc.get("com.atproto.repo.getRecord", { ··· 143 147 return account; 144 148 } catch (e) { 145 149 console.error(`Error fetching metadata for ${did}:`, e); 146 - return { 147 - did: "error", 148 - displayName: "", 149 - avatarCid: null, 150 - handle: "error", 151 - }; 150 + return null; 152 151 } 153 152 }; 154 153 ··· 157 156 const metadata = await Promise.all( 158 157 dids.map(async (repo: `did:${string}:${string}`) => { 159 158 return await getAccountMetadata(repo); 160 - }) 159 + }), 161 160 ); 162 - return metadata.filter((account) => account.did !== "error"); 163 - }; 164 - 165 - const fetchPosts = async (did: string) => { 166 - try { 167 - const { data } = await rpc.get("com.atproto.repo.listRecords", { 168 - params: { 169 - repo: did as At.Identifier, 170 - collection: "app.bsky.feed.post", 171 - limit: Config.MAX_POSTS, 172 - }, 173 - }); 174 - return { 175 - records: data.records as ComAtprotoRepoListRecords.Record[], 176 - did: did, 177 - error: false, 178 - }; 179 - } catch (e) { 180 - console.error(`Error fetching posts for ${did}:`, e); 181 - return { 182 - records: [], 183 - did: did, 184 - error: true, 185 - }; 186 - } 161 + return metadata.filter((account) => account !== null) as AccountMetadata[]; 187 162 }; 188 163 189 164 const identityResolve = async (did: At.Did) => { ··· 196 171 197 172 if (did.startsWith("did:plc:") || did.startsWith("did:web:")) { 198 173 const doc = await resolver.resolve( 199 - did as `did:plc:${string}` | `did:web:${string}` 174 + did as `did:plc:${string}` | `did:web:${string}`, 200 175 ); 201 176 return doc; 202 177 } else { ··· 219 194 } 220 195 }; 221 196 222 - const fetchAllPosts = async () => { 223 - const users: AccountMetadata[] = await getAllMetadataFromPds(); 224 - const postRecords = await Promise.all( 225 - users.map( 226 - async (metadata: AccountMetadata) => await fetchPosts(metadata.did) 227 - ) 228 - ); 229 - const validPostRecords = postRecords.filter((record) => !record.error); 230 - const posts: Post[] = validPostRecords.flatMap((userFetch) => 231 - userFetch.records.map((record) => { 232 - const user = users.find( 233 - (user: AccountMetadata) => user.did == userFetch.did 197 + interface PostsAcc { 198 + posts: ComAtprotoRepoListRecords.Record[]; 199 + account: AccountMetadata; 200 + } 201 + const getCutoffDate = (postAccounts: PostsAcc[]) => { 202 + const now = Date.now(); 203 + let cutoffDate: Date | null = null; 204 + postAccounts.forEach((postAcc) => { 205 + const latestPost = new Date( 206 + (postAcc.posts[postAcc.posts.length - 1].value as AppBskyFeedPost.Record) 207 + .createdAt, 208 + ); 209 + if (!cutoffDate) { 210 + cutoffDate = latestPost; 211 + } else { 212 + if (latestPost > cutoffDate) { 213 + cutoffDate = latestPost; 214 + } 215 + } 216 + }); 217 + if (cutoffDate) { 218 + return cutoffDate; 219 + } else { 220 + return new Date(now); 221 + } 222 + }; 223 + 224 + const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => { 225 + // filter posts for each account that are older than the cutoff date and save the cursor of the last post included 226 + const filteredPosts: PostsAcc[] = posts.map((postAcc) => { 227 + const filtered = postAcc.posts.filter((post) => { 228 + const postDate = new Date( 229 + (post.value as AppBskyFeedPost.Record).createdAt, 230 + ); 231 + return postDate >= cutoffDate; 232 + }); 233 + if (filtered.length > 0) { 234 + postAcc.account.currentCursor = processAtUri(filtered[filtered.length - 1].uri).rkey; 235 + } 236 + return { 237 + posts: filtered, 238 + account: postAcc.account, 239 + }; 240 + }); 241 + return filteredPosts; 242 + }; 243 + // nightmare function. However it works so I am not touching it 244 + const getNextPosts = async () => { 245 + if (!accountsMetadata.length) { 246 + accountsMetadata = await getAllMetadataFromPds(); 247 + } 248 + 249 + const postsAcc: PostsAcc[] = await Promise.all( 250 + accountsMetadata.map(async (account) => { 251 + const posts = await fetchPostsForUser( 252 + account.did, 253 + account.currentCursor || null, 234 254 ); 235 - if (!user) { 236 - throw new Error(`User with DID ${userFetch.did} not found`); 255 + if (posts) { 256 + return { 257 + posts: posts, 258 + account: account, 259 + }; 260 + } else { 261 + return { 262 + posts: [], 263 + account: account, 264 + }; 237 265 } 238 - return new Post(record, user); 239 - }) 266 + }), 267 + ); 268 + const recordsFiltered = postsAcc.filter((postAcc) => 269 + postAcc.posts.length > 0 240 270 ); 241 - 242 - posts.sort((a, b) => b.timestamp - a.timestamp); 243 - 244 - if(!Config.SHOW_FUTURE_POSTS) { 245 - // Filter out posts that are in the future 271 + const cutoffDate = getCutoffDate(recordsFiltered); 272 + const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate); 273 + // update the accountMetadata with the new cursor 274 + accountsMetadata = accountsMetadata.map((account) => { 275 + const postAcc = recordsCutoff.find( 276 + (postAcc) => postAcc.account.did == account.did, 277 + ); 278 + if (postAcc) { 279 + account.currentCursor = postAcc.account.currentCursor; 280 + } 281 + return account; 282 + } 283 + ); 284 + // throw the records in a big single array 285 + let records = recordsCutoff.flatMap((postAcc) => postAcc.posts); 286 + // sort the records by timestamp 287 + records = records.sort((a, b) => { 288 + const aDate = new Date( 289 + (a.value as AppBskyFeedPost.Record).createdAt, 290 + ).getTime(); 291 + const bDate = new Date( 292 + (b.value as AppBskyFeedPost.Record).createdAt, 293 + ).getTime(); 294 + return bDate - aDate; 295 + }); 296 + // filter out posts that are in the future 297 + if (!Config.SHOW_FUTURE_POSTS) { 246 298 const now = Date.now(); 247 - const filteredPosts = posts.filter((post) => post.timestamp <= now); 248 - return filteredPosts.slice(0, Config.MAX_POSTS); 299 + records = records.filter((post) => { 300 + const postDate = new Date( 301 + (post.value as AppBskyFeedPost.Record).createdAt, 302 + ).getTime(); 303 + return postDate <= now; 304 + }); 249 305 } 250 306 251 - return posts.slice(0, Config.MAX_POSTS); 307 + const newPosts = records.map((record) => { 308 + const account = accountsMetadata.find( 309 + (account) => account.did == processAtUri(record.uri).repo, 310 + ); 311 + if (!account) { 312 + throw new Error( 313 + `Account with DID ${processAtUri(record.uri).repo} not found`, 314 + ); 315 + } 316 + return new Post(record, account); 317 + }); 318 + return newPosts; 252 319 }; 253 - export { fetchAllPosts, getAllMetadataFromPds, Post }; 320 + 321 + const fetchPostsForUser = async (did: At.Did, cursor: string | null) => { 322 + try { 323 + const { data } = await rpc.get("com.atproto.repo.listRecords", { 324 + params: { 325 + repo: did as At.Identifier, 326 + collection: "app.bsky.feed.post", 327 + limit: Config.MAX_POSTS, 328 + cursor: cursor || undefined, 329 + }, 330 + }); 331 + return data.records as ComAtprotoRepoListRecords.Record[]; 332 + } catch (e) { 333 + console.error(`Error fetching posts for ${did}:`, e); 334 + return null; 335 + } 336 + }; 337 + 338 + export { getAllMetadataFromPds, getNextPosts, Post }; 254 339 export type { AccountMetadata };