whey.party / skylite
an attempt to make a lightweight, easily self-hostable, scoped bluesky appview
fork atom

partial partials

rimar1337 b91eeceb efe3c430

+158 -70
unified split
+157 -69
indexserver.ts
··· 12 12 import * as ATPAPI from "npm:@atproto/api"; 13 13 import { AtUri } from "npm:@atproto/api"; 14 14 import * as IndexServerAPI from "./indexclient/index.ts"; 15 - import * as IndexServerUtils from "./indexclient/util.ts" 15 + import * as IndexServerUtils from "./indexclient/util.ts"; 16 + import { isPostView } from "./indexclient/types/app/bsky/feed/defs.ts"; 16 17 17 18 export interface IndexServerConfig { 18 19 baseDbPath: string; ··· 273 274 274 275 // TODO: not partial yet, currently skips refs 275 276 276 - const qresult = this.queryActorLikesPartial(jsonTyped.actor, jsonTyped.cursor); 277 + const qresult = this.queryActorLikesPartial( 278 + jsonTyped.actor, 279 + jsonTyped.cursor 280 + ); 277 281 if (!qresult) { 278 282 return new Response( 279 283 JSON.stringify({ ··· 302 306 303 307 // TODO: not partial yet, currently skips refs 304 308 305 - const qresult = this.queryAuthorFeed(jsonTyped.actor, jsonTyped.cursor); 309 + const qresult = this.queryAuthorFeedPartial(jsonTyped.actor, jsonTyped.cursor); 306 310 if (!qresult) { 307 311 return new Response( 308 312 JSON.stringify({ ··· 359 363 360 364 // TODO: not partial yet, currently skips refs 361 365 362 - const qresult = this.queryPostThread(jsonTyped.uri); 366 + const qresult = this.queryPostThreadPartial(jsonTyped.uri); 363 367 if (!qresult) { 364 368 return new Response( 365 369 JSON.stringify({ ··· 1032 1036 1033 1037 return post; 1034 1038 } 1035 - 1036 - constructPostViewRef(uri: string): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef { 1039 + 1040 + constructPostViewRef( 1041 + uri: string 1042 + ): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef { 1037 1043 const post: IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef = { 1038 1044 uri: uri, 1039 1045 cid: "cid.invalid", // oh shit we dont know the cid TODO: major design flaw ··· 1063 1069 ): IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef { 1064 1070 const post = this.constructPostViewRef(uri); 1065 1071 1066 - const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = { 1067 - $type: "party.whey.app.bsky.feed.defs#feedViewPostRef", 1068 - post: post as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, 1069 - } 1072 + const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = 1073 + { 1074 + $type: "party.whey.app.bsky.feed.defs#feedViewPostRef", 1075 + post: post as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, 1076 + }; 1070 1077 1071 - return feedviewpostref 1078 + return feedviewpostref; 1072 1079 } 1073 1080 1074 1081 // user feedgens ··· 1181 1188 1182 1189 // user feeds 1183 1190 1184 - queryAuthorFeed( 1191 + queryAuthorFeedPartial( 1185 1192 did: string, 1186 1193 cursor?: string 1187 1194 ): 1188 1195 | { 1189 - items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1196 + items: ( 1197 + | ATPAPI.AppBskyFeedDefs.FeedViewPost 1198 + | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef 1199 + )[]; 1190 1200 cursor: string | undefined; 1191 1201 } 1192 1202 | undefined { ··· 1194 1204 const db = this.userManager.getDbForDid(did); 1195 1205 if (!db) return; 1196 1206 1197 - // TODO: implement this for real 1198 - let query = ` 1199 - SELECT uri, indexedat, cid 1200 - FROM app_bsky_feed_post 1201 - WHERE did = ? 1202 - `; 1203 - const params: (string | number)[] = [did]; 1207 + const subquery = ` 1208 + SELECT uri, cid, indexedat, 'post' as type, null as subject 1209 + FROM app_bsky_feed_post 1210 + WHERE did = ? 1211 + UNION ALL 1212 + SELECT uri, cid, indexedat, 'repost' as type, subject 1213 + FROM app_bsky_feed_repost 1214 + WHERE did = ? 1215 + `; 1216 + 1217 + let query = `SELECT * FROM (${subquery}) as feed_items`; 1218 + const params: (string | number)[] = [did, did]; 1204 1219 1205 1220 if (cursor) { 1206 1221 const [indexedat, cid] = cursor.split("::"); 1207 - query += ` AND (indexedat < ? OR (indexedat = ? AND cid < ?))`; 1222 + query += ` WHERE (indexedat < ? OR (indexedat = ? AND cid < ?))`; 1208 1223 params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); 1209 1224 } 1210 1225 ··· 1215 1230 uri: string; 1216 1231 indexedat: number; 1217 1232 cid: string; 1233 + type: "post" | "repost"; 1234 + subject: string | null; 1218 1235 }[]; 1236 + 1237 + const authorProfile = this.queryProfileView(did,"Basic"); 1219 1238 1220 1239 const items = rows 1221 - .map((row) => this.queryFeedViewPost(row.uri)) // TODO: for replies and repost i should inject the reason here 1240 + .map((row) => { 1241 + if (row.type === "repost" && row.subject) { 1242 + const subjectDid = new AtUri(row.subject).host 1243 + 1244 + const originalPost = this.handlesDid(subjectDid) 1245 + ? this.queryFeedViewPost(row.subject) 1246 + : this.constructFeedViewPostRef(row.subject); 1247 + 1248 + if (!originalPost || !authorProfile) return null; 1249 + 1250 + return { 1251 + post: originalPost, 1252 + reason: { 1253 + $type: "app.bsky.feed.defs#reasonRepost", 1254 + by: authorProfile, 1255 + indexedAt: new Date(row.indexedat).toISOString(), 1256 + }, 1257 + }; 1258 + } else { 1259 + return this.queryFeedViewPost(row.uri); 1260 + } 1261 + }) 1222 1262 .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1223 1263 1224 1264 const lastItem = rows[rows.length - 1]; ··· 1246 1286 cursor?: string 1247 1287 ): 1248 1288 | { 1249 - items: (ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef)[]; 1289 + items: ( 1290 + | ATPAPI.AppBskyFeedDefs.FeedViewPost 1291 + | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef 1292 + )[]; 1250 1293 cursor: string | undefined; 1251 1294 } 1252 1295 | undefined { ··· 1279 1322 1280 1323 const items = rows 1281 1324 .map((row) => { 1282 - const subjectDid = new AtUri(row.subject).host; 1325 + const subjectDid = new AtUri(row.subject).host; 1283 1326 1284 1327 if (this.handlesDid(subjectDid)) { 1285 1328 return this.queryFeedViewPost(row.subject); ··· 1287 1330 return this.constructFeedViewPostRef(row.subject); 1288 1331 } 1289 1332 }) 1290 - .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef => !!p); 1333 + .filter( 1334 + ( 1335 + p 1336 + ): p is 1337 + | ATPAPI.AppBskyFeedDefs.FeedViewPost 1338 + | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef => !!p 1339 + ); 1291 1340 1292 1341 const lastItem = rows[rows.length - 1]; 1293 1342 const nextCursor = lastItem ··· 1371 1420 .map((row) => this.queryFeedViewPost(row.srcuri)) 1372 1421 .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1373 1422 } 1374 - 1375 - queryPostThread( 1423 + _getPostViewUnion( 1376 1424 uri: string 1377 - ): ATPAPI.AppBskyFeedGetPostThread.OutputSchema | undefined { 1378 - const post = this.queryPostView(uri); 1425 + ): 1426 + | ATPAPI.AppBskyFeedDefs.PostView 1427 + | IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef 1428 + | undefined { 1429 + try { 1430 + const postDid = new AtUri(uri).hostname; 1431 + if (this.handlesDid(postDid)) { 1432 + return this.queryPostView(uri); 1433 + } else { 1434 + return this.constructPostViewRef(uri); 1435 + } 1436 + } catch (_e) { 1437 + return undefined; 1438 + } 1439 + } 1440 + queryPostThreadPartial( 1441 + uri: string 1442 + ): IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.OutputSchema | undefined { 1443 + 1444 + const post = this._getPostViewUnion(uri); 1445 + 1379 1446 if (!post) { 1380 1447 return { 1381 1448 thread: { ··· 1386 1453 }; 1387 1454 } 1388 1455 1389 - const thread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1390 - $type: "app.bsky.feed.defs#threadViewPost", 1391 - post: post, 1456 + const thread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { 1457 + $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", 1458 + post: post as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView> | IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, 1392 1459 replies: [], 1393 1460 }; 1394 1461 1395 1462 let current = thread; 1396 - while ((current.post.record.reply as any)?.parent?.uri) { 1397 - const parentUri = (current.post.record.reply as any)?.parent?.uri; 1398 - const parentPost = this.queryPostView(parentUri); 1399 - if (!parentPost) break; 1463 + // we can only climb the parent tree if we have the full post record. 1464 + // which is not implemented yet (sad i know) 1465 + if (isPostView(current.post) && isFeedPostRecord(current.post.record) && current.post.record?.reply?.parent?.uri) { 1466 + let parentUri: string | undefined = current.post.record.reply.parent.uri; 1467 + 1468 + // keep climbing as long as we find a valid parent post. 1469 + while (parentUri) { 1470 + const parentPost = this._getPostViewUnion(parentUri); 1471 + if (!parentPost) break; // stop if a parent in the chain is not found. 1472 + 1473 + const parentThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { 1474 + $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", 1475 + post: parentPost as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView>, 1476 + replies: [current as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>], 1477 + }; 1478 + current.parent = parentThread as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>; 1479 + current = parentThread; 1400 1480 1401 - const parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1402 - $type: "app.bsky.feed.defs#threadViewPost", 1403 - post: parentPost, 1404 - replies: [ 1405 - current as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>, 1406 - ], 1407 - }; 1408 - current.parent = 1409 - parentThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>; 1410 - current = parentThread; 1481 + // check if the new current post has a parent to continue the loop 1482 + parentUri = (isPostView(current.post) && isFeedPostRecord(current.post.record)) ? current.post.record?.reply?.parent?.uri : undefined; 1483 + } 1411 1484 } 1412 1485 1486 + 1487 + 1413 1488 const seenUris = new Set<string>(); 1414 - const fetchReplies = ( 1415 - parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost 1416 - ) => { 1489 + const fetchReplies = (parentThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef) => { 1490 + if (!parentThread.post || !('uri' in parentThread.post)) { 1491 + return; 1492 + } 1417 1493 if (seenUris.has(parentThread.post.uri)) return; 1418 1494 seenUris.add(parentThread.post.uri); 1419 1495 1420 1496 const parentUri = new AtUri(parentThread.post.uri); 1421 1497 const parentAuthorDid = parentUri.hostname; 1498 + 1499 + // replies can only be discovered for local posts where we have the backlink data 1500 + if (!this.handlesDid(parentAuthorDid)) return; 1501 + 1422 1502 const db = this.userManager.getDbForDid(parentAuthorDid); 1423 1503 if (!db) return; 1424 1504 1425 1505 const stmt = db.prepare(` 1426 - SELECT srcuri 1427 - FROM backlink_skeleton 1428 - WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'replyparent' 1429 - `); 1506 + SELECT srcuri 1507 + FROM backlink_skeleton 1508 + WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'replyparent' 1509 + `); 1430 1510 const replyRows = stmt.all(parentThread.post.uri) as { srcuri: string }[]; 1431 1511 1432 1512 const replies = replyRows 1433 - .map((row) => this.queryPostView(row.srcuri)) 1434 - .filter((p): p is ATPAPI.AppBskyFeedDefs.PostView => !!p); 1513 + .map((row) => this._getPostViewUnion(row.srcuri)) 1514 + .filter((p): p is ATPAPI.AppBskyFeedDefs.PostView | IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef => !!p); 1435 1515 1436 1516 for (const replyPost of replies) { 1437 - const replyThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1438 - $type: "app.bsky.feed.defs#threadViewPost", 1439 - post: replyPost, 1440 - parent: 1441 - parentThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>, 1517 + const replyThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { 1518 + $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", 1519 + post: replyPost as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView> | IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, 1520 + parent: parentThread as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>, 1442 1521 replies: [], 1443 1522 }; 1444 - parentThread.replies?.push( 1445 - replyThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost> 1446 - ); 1447 - fetchReplies(replyThread); 1523 + parentThread.replies?.push(replyThread as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>); 1524 + fetchReplies(replyThread); // recurse 1448 1525 } 1449 1526 }; 1450 1527 1451 1528 fetchReplies(thread); 1452 1529 1453 - const returned = 1454 - thread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>; 1530 + const returned = current as unknown as IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef; 1455 1531 1456 - return { thread: returned }; 1532 + return { thread: returned as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef> }; 1457 1533 } 1534 + 1458 1535 1459 1536 /** 1460 1537 * please do not use this, use openDbForDid() instead ··· 1470 1547 } 1471 1548 /** 1472 1549 * @deprecated use handlesDid() instead 1473 - * @param did 1474 - * @returns 1550 + * @param did 1551 + * @returns 1475 1552 */ 1476 1553 isRegisteredIndexUser(did: string): boolean { 1477 1554 const stmt = this.systemDB.prepare(` ··· 1888 1965 1889 1966 export function isDid(str: string): boolean { 1890 1967 return typeof str === "string" && str.startsWith("did:"); 1968 + } 1969 + 1970 + function isFeedPostRecord( 1971 + post: unknown 1972 + ): post is ATPAPI.AppBskyFeedPost.Record { 1973 + return ( 1974 + typeof post === "object" && 1975 + post !== null && 1976 + "$type" in post && 1977 + (post as any).$type === "app.bsky.feed.post" 1978 + ); 1891 1979 } 1892 1980 1893 1981 function isImageEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedImages.Main {
+1 -1
readme.md
··· 7 7 - the server stuff: [sqlite](https://jsr.io/@db/sqlite) db, typescript with [codegen](https://www.npmjs.com/package/@atproto/lex-cli), and [deno](https://deno.com/) 8 8 9 9 ## Status 10 - (as of 25 aug 2025) 10 + (as of 26 aug 2025) 11 11 currently the state of the project is: 12 12 ### Index Server 13 13 - Database: