even more intense index server instantiation · whey.party/skylite@dae503d

+1463 -1395

indexserver.ts

··· 5 5 import * as IndexServerTypes from "./utils/indexservertypes.ts"; 6 6 import { Database } from "jsr:@db/sqlite@0.11"; 7 7 import { setupUserDb } from "./utils/dbuser.ts"; 8 - import { indexerUserManager, jetstreamurl, systemDB } from "./main.ts"; 8 + // import { systemDB } from "./main.ts"; 9 9 import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts"; 10 10 import { handleSpacedust, SpacedustLinkMessage } from "./index/spacedust.ts"; 11 11 import { handleJetstream } from "./index/jetstream.ts"; ··· 13 13 import { AtUri } from "npm:@atproto/api"; 14 14 import * as IndexServerAPI from "./indexclient/index.ts"; 15 15 16 + export interface IndexServerConfig { 17 + baseDbPath: string; 18 + systemDbPath: string; 19 + jetstreamUrl: string; 20 + } 21 + 22 + interface BaseRow { 23 + uri: string; 24 + did: string; 25 + cid: string | null; 26 + rev: string | null; 27 + createdat: number | null; 28 + indexedat: number; 29 + json: string | null; 30 + } 31 + interface GeneratorRow extends BaseRow { 32 + displayname: string | null; 33 + description: string | null; 34 + avatarcid: string | null; 35 + } 36 + interface LikeRow extends BaseRow { 37 + subject: string; 38 + } 39 + interface RepostRow extends BaseRow { 40 + subject: string; 41 + } 42 + interface BacklinkRow { 43 + srcuri: string; 44 + srcdid: string; 45 + } 46 + 47 + const FEED_LIMIT = 50; 48 + 49 + export class IndexServer { 50 + private config: IndexServerConfig; 51 + public userManager: IndexServerUserManager; 52 + public systemDB: Database; 53 + 54 + constructor(config: IndexServerConfig) { 55 + this.config = config; 56 + 57 + // We will initialize the system DB and user manager here 58 + this.systemDB = new Database(this.config.systemDbPath); 59 + // TODO: We need to setup the system DB schema if it's new 60 + 61 + this.userManager = new IndexServerUserManager(this); // Pass the server instance 62 + } 63 + 64 + public start() { 65 + // This is where we'll kick things off, like the cold start 66 + this.userManager.coldStart(this.systemDB); 67 + console.log("IndexServer started."); 68 + } 69 + 70 + public async handleRequest(req: Request): Promise<Response> { 71 + const url = new URL(req.url); 72 + // We will add routing logic here later to call our handlers 73 + if (url.pathname.startsWith("/xrpc/")) { 74 + return this.indexServerHandler(req); 75 + } 76 + if (url.pathname.startsWith("/links")) { 77 + return this.constellationAPIHandler(req); 78 + } 79 + return new Response("Not Found", { status: 404 }); 80 + } 81 + 82 + // We will move all the global functions into this class as methods... 83 + indexServerHandler(req: Request): Response { 84 + const url = new URL(req.url); 85 + const pathname = url.pathname; 86 + //const bskyUrl = `https://api.bsky.app${pathname}${url.search}`; 87 + //const hasAuth = req.headers.has("authorization"); 88 + const xrpcMethod = pathname.startsWith("/xrpc/") 89 + ? pathname.slice("/xrpc/".length) 90 + : null; 91 + const searchParams = searchParamsToJson(url.searchParams); 92 + console.log(JSON.stringify(searchParams, null, 2)); 93 + const jsonUntyped = searchParams; 94 + 95 + switch (xrpcMethod) { 96 + case "app.bsky.actor.getProfile": { 97 + const jsonTyped = 98 + jsonUntyped as IndexServerTypes.AppBskyActorGetProfile.QueryParams; 99 + 100 + const res = this.queryProfileView(jsonTyped.actor, "Detailed"); 101 + if (!res) 102 + return new Response( 103 + JSON.stringify({ 104 + error: "User not found", 105 + }), 106 + { 107 + status: 404, 108 + headers: withCors({ "Content-Type": "application/json" }), 109 + } 110 + ); 111 + const response: IndexServerTypes.AppBskyActorGetProfile.OutputSchema = 112 + res; 113 + 114 + return new Response(JSON.stringify(response), { 115 + headers: withCors({ "Content-Type": "application/json" }), 116 + }); 117 + } 118 + case "app.bsky.actor.getProfiles": { 119 + const jsonTyped = 120 + jsonUntyped as IndexServerTypes.AppBskyActorGetProfiles.QueryParams; 121 + 122 + if (typeof jsonUntyped?.actors === "string") { 123 + const res = this.queryProfileView( 124 + jsonUntyped.actors as string, 125 + "Detailed" 126 + ); 127 + if (!res) 128 + return new Response( 129 + JSON.stringify({ 130 + error: "User not found", 131 + }), 132 + { 133 + status: 404, 134 + headers: withCors({ "Content-Type": "application/json" }), 135 + } 136 + ); 137 + const response: IndexServerTypes.AppBskyActorGetProfiles.OutputSchema = 138 + { 139 + profiles: [res], 140 + }; 141 + 142 + return new Response(JSON.stringify(response), { 143 + headers: withCors({ "Content-Type": "application/json" }), 144 + }); 145 + } 146 + 147 + const res: ATPAPI.AppBskyActorDefs.ProfileViewDetailed[] = 148 + jsonTyped.actors 149 + .map((actor) => { 150 + return this.queryProfileView(actor, "Detailed"); 151 + }) 152 + .filter( 153 + (x): x is ATPAPI.AppBskyActorDefs.ProfileViewDetailed => 154 + x !== undefined 155 + ); 156 + 157 + if (!res) 158 + return new Response( 159 + JSON.stringify({ 160 + error: "User not found", 161 + }), 162 + { 163 + status: 404, 164 + headers: withCors({ "Content-Type": "application/json" }), 165 + } 166 + ); 167 + 168 + const response: IndexServerTypes.AppBskyActorGetProfiles.OutputSchema = 169 + { 170 + profiles: res, 171 + }; 172 + 173 + return new Response(JSON.stringify(response), { 174 + headers: withCors({ "Content-Type": "application/json" }), 175 + }); 176 + } 177 + case "app.bsky.feed.getActorFeeds": { 178 + const jsonTyped = 179 + jsonUntyped as IndexServerTypes.AppBskyFeedGetActorFeeds.QueryParams; 180 + 181 + const qresult = this.queryActorFeeds(jsonTyped.actor); 182 + 183 + const response: IndexServerTypes.AppBskyFeedGetActorFeeds.OutputSchema = 184 + { 185 + feeds: qresult, 186 + }; 187 + 188 + return new Response(JSON.stringify(response), { 189 + headers: withCors({ "Content-Type": "application/json" }), 190 + }); 191 + } 192 + case "app.bsky.feed.getFeedGenerator": { 193 + const jsonTyped = 194 + jsonUntyped as IndexServerTypes.AppBskyFeedGetFeedGenerator.QueryParams; 195 + 196 + const qresult = this.queryFeedGenerator(jsonTyped.feed); 197 + if (!qresult) { 198 + return new Response( 199 + JSON.stringify({ 200 + error: "Feed not found", 201 + }), 202 + { 203 + status: 404, 204 + headers: withCors({ "Content-Type": "application/json" }), 205 + } 206 + ); 207 + } 208 + 209 + const response: IndexServerTypes.AppBskyFeedGetFeedGenerator.OutputSchema = 210 + { 211 + view: qresult, 212 + isOnline: true, // lmao 213 + isValid: true, // lmao 214 + }; 215 + 216 + return new Response(JSON.stringify(response), { 217 + headers: withCors({ "Content-Type": "application/json" }), 218 + }); 219 + } 220 + case "app.bsky.feed.getFeedGenerators": { 221 + const jsonTyped = 222 + jsonUntyped as IndexServerTypes.AppBskyFeedGetFeedGenerators.QueryParams; 223 + 224 + const qresult = this.queryFeedGenerators(jsonTyped.feeds); 225 + if (!qresult) { 226 + return new Response( 227 + JSON.stringify({ 228 + error: "Feed not found", 229 + }), 230 + { 231 + status: 404, 232 + headers: withCors({ "Content-Type": "application/json" }), 233 + } 234 + ); 235 + } 236 + 237 + const response: IndexServerTypes.AppBskyFeedGetFeedGenerators.OutputSchema = 238 + { 239 + feeds: qresult, 240 + }; 241 + 242 + return new Response(JSON.stringify(response), { 243 + headers: withCors({ "Content-Type": "application/json" }), 244 + }); 245 + } 246 + case "app.bsky.feed.getPosts": { 247 + const jsonTyped = 248 + jsonUntyped as IndexServerTypes.AppBskyFeedGetPosts.QueryParams; 249 + 250 + const posts: IndexServerTypes.AppBskyFeedGetPosts.OutputSchema["posts"] = 251 + jsonTyped.uris 252 + .map((uri) => { 253 + return this.queryPostView(uri); 254 + }) 255 + .filter(Boolean) as ATPAPI.AppBskyFeedDefs.PostView[]; 256 + 257 + const response: IndexServerTypes.AppBskyFeedGetPosts.OutputSchema = { 258 + posts, 259 + }; 260 + 261 + return new Response(JSON.stringify(response), { 262 + headers: withCors({ "Content-Type": "application/json" }), 263 + }); 264 + } 265 + case "party.whey.app.bsky.feed.getActorLikesPartial": { 266 + const jsonTyped = 267 + jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetActorLikesPartial.QueryParams; 268 + 269 + // TODO: not partial yet, currently skips refs 270 + 271 + const qresult = this.queryActorLikes(jsonTyped.actor, jsonTyped.cursor); 272 + if (!qresult) { 273 + return new Response( 274 + JSON.stringify({ 275 + error: "Feed not found", 276 + }), 277 + { 278 + status: 404, 279 + headers: withCors({ "Content-Type": "application/json" }), 280 + } 281 + ); 282 + } 283 + 284 + const response: IndexServerTypes.PartyWheyAppBskyFeedGetActorLikesPartial.OutputSchema = 285 + { 286 + feed: qresult.items as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.FeedViewPost>[], 287 + cursor: qresult.cursor, 288 + }; 289 + 290 + return new Response(JSON.stringify(response), { 291 + headers: withCors({ "Content-Type": "application/json" }), 292 + }); 293 + } 294 + case "party.whey.app.bsky.feed.getAuthorFeedPartial": { 295 + const jsonTyped = 296 + jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetAuthorFeedPartial.QueryParams; 297 + 298 + // TODO: not partial yet, currently skips refs 299 + 300 + const qresult = this.queryAuthorFeed(jsonTyped.actor, jsonTyped.cursor); 301 + if (!qresult) { 302 + return new Response( 303 + JSON.stringify({ 304 + error: "Feed not found", 305 + }), 306 + { 307 + status: 404, 308 + headers: withCors({ "Content-Type": "application/json" }), 309 + } 310 + ); 311 + } 312 + 313 + const response: IndexServerTypes.PartyWheyAppBskyFeedGetAuthorFeedPartial.OutputSchema = 314 + { 315 + feed: qresult.items as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.FeedViewPost>[], 316 + cursor: qresult.cursor, 317 + }; 318 + 319 + return new Response(JSON.stringify(response), { 320 + headers: withCors({ "Content-Type": "application/json" }), 321 + }); 322 + } 323 + case "party.whey.app.bsky.feed.getLikesPartial": { 324 + const jsonTyped = 325 + jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetLikesPartial.QueryParams; 326 + 327 + // TODO: not partial yet, currently skips refs 328 + 329 + const qresult = this.queryLikes(jsonTyped.uri); 330 + if (!qresult) { 331 + return new Response( 332 + JSON.stringify({ 333 + error: "Feed not found", 334 + }), 335 + { 336 + status: 404, 337 + headers: withCors({ "Content-Type": "application/json" }), 338 + } 339 + ); 340 + } 341 + const response: IndexServerTypes.PartyWheyAppBskyFeedGetLikesPartial.OutputSchema = 342 + { 343 + // @ts-ignore whatever i dont care TODO: fix ts ignores 344 + likes: qresult, 345 + }; 346 + 347 + return new Response(JSON.stringify(response), { 348 + headers: withCors({ "Content-Type": "application/json" }), 349 + }); 350 + } 351 + case "party.whey.app.bsky.feed.getPostThreadPartial": { 352 + const jsonTyped = 353 + jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.QueryParams; 354 + 355 + // TODO: not partial yet, currently skips refs 356 + 357 + const qresult = this.queryPostThread(jsonTyped.uri); 358 + if (!qresult) { 359 + return new Response( 360 + JSON.stringify({ 361 + error: "Feed not found", 362 + }), 363 + { 364 + status: 404, 365 + headers: withCors({ "Content-Type": "application/json" }), 366 + } 367 + ); 368 + } 369 + const response: IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.OutputSchema = 370 + qresult; 371 + 372 + return new Response(JSON.stringify(response), { 373 + headers: withCors({ "Content-Type": "application/json" }), 374 + }); 375 + } 376 + case "party.whey.app.bsky.feed.getQuotesPartial": { 377 + const jsonTyped = 378 + jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetQuotesPartial.QueryParams; 379 + 380 + // TODO: not partial yet, currently skips refs 381 + 382 + const qresult = this.queryQuotes(jsonTyped.uri); 383 + if (!qresult) { 384 + return new Response( 385 + JSON.stringify({ 386 + error: "Feed not found", 387 + }), 388 + { 389 + status: 404, 390 + headers: withCors({ "Content-Type": "application/json" }), 391 + } 392 + ); 393 + } 394 + const response: IndexServerTypes.PartyWheyAppBskyFeedGetQuotesPartial.OutputSchema = 395 + { 396 + uri: jsonTyped.uri, 397 + posts: qresult.map((feedviewpost) => { 398 + return feedviewpost.post as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView>; 399 + }), 400 + }; 401 + 402 + return new Response(JSON.stringify(response), { 403 + headers: withCors({ "Content-Type": "application/json" }), 404 + }); 405 + } 406 + case "party.whey.app.bsky.feed.getRepostedByPartial": { 407 + const jsonTyped = 408 + jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetRepostedByPartial.QueryParams; 409 + 410 + // TODO: not partial yet, currently skips refs 411 + 412 + const qresult = this.queryReposts(jsonTyped.uri); 413 + if (!qresult) { 414 + return new Response( 415 + JSON.stringify({ 416 + error: "Feed not found", 417 + }), 418 + { 419 + status: 404, 420 + headers: withCors({ "Content-Type": "application/json" }), 421 + } 422 + ); 423 + } 424 + const response: IndexServerTypes.PartyWheyAppBskyFeedGetRepostedByPartial.OutputSchema = 425 + { 426 + uri: jsonTyped.uri, 427 + repostedBy: 428 + qresult as ATPAPI.$Typed<ATPAPI.AppBskyActorDefs.ProfileView>[], 429 + }; 430 + 431 + return new Response(JSON.stringify(response), { 432 + headers: withCors({ "Content-Type": "application/json" }), 433 + }); 434 + } 435 + // TODO: too hard for now 436 + // case "party.whey.app.bsky.feed.getListFeedPartial": { 437 + // const jsonTyped = 438 + // jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetListFeedPartial.QueryParams; 439 + 440 + // const response: IndexServerTypes.PartyWheyAppBskyFeedGetListFeedPartial.OutputSchema = 441 + // {}; 442 + 443 + // return new Response(JSON.stringify(response), { 444 + // headers: withCors({ "Content-Type": "application/json" }), 445 + // }); 446 + // } 447 + /* three more coming soon 448 + app.bsky.graph.getLists 449 + app.bsky.graph.getList 450 + app.bsky.graph.getActorStarterPacks 451 + */ 452 + default: { 453 + return new Response( 454 + JSON.stringify({ 455 + error: "XRPCNotSupported", 456 + message: 457 + "HEY hello there my name is whey dot party and you have used my custom appview that is very cool but have you considered that XRPC Not Supported", 458 + }), 459 + { 460 + status: 404, 461 + headers: withCors({ "Content-Type": "application/json" }), 462 + } 463 + ); 464 + } 465 + } 466 + 467 + // return new Response("Not Found", { status: 404 }); 468 + } 469 + 470 + constellationAPIHandler(req: Request): Response { 471 + const url = new URL(req.url); 472 + const pathname = url.pathname; 473 + const searchParams = searchParamsToJson(url.searchParams) as linksQuery; 474 + const jsonUntyped = searchParams; 475 + 476 + if (!jsonUntyped.target) { 477 + return new Response( 478 + JSON.stringify({ error: "Missing required parameter: target" }), 479 + { 480 + status: 400, 481 + headers: withCors({ "Content-Type": "application/json" }), 482 + } 483 + ); 484 + } 485 + 486 + const did = isDid(searchParams.target) 487 + ? searchParams.target 488 + : new AtUri(searchParams.target).host; 489 + const db = this.userManager.getDbForDid(did); 490 + if (!db) { 491 + return new Response( 492 + JSON.stringify({ 493 + error: "User not found", 494 + }), 495 + { 496 + status: 404, 497 + headers: withCors({ "Content-Type": "application/json" }), 498 + } 499 + ); 500 + } 501 + 502 + const limit = 16; //Math.min(parseInt(searchParams.limit || "50", 10), 100); 503 + const offset = parseInt(searchParams.cursor || "0", 10); 504 + 505 + switch (pathname) { 506 + case "/links": { 507 + const jsonTyped = jsonUntyped as linksQuery; 508 + if (!jsonTyped.collection || !jsonTyped.path) { 509 + return new Response( 510 + JSON.stringify({ 511 + error: "Missing required parameters: collection, path", 512 + }), 513 + { 514 + status: 400, 515 + headers: withCors({ "Content-Type": "application/json" }), 516 + } 517 + ); 518 + } 519 + 520 + const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 521 + /^\./, 522 + "" 523 + )}`; 524 + 525 + const paginatedSql = `${SQL.links} LIMIT ? OFFSET ?`; 526 + const rows = db 527 + .prepare(paginatedSql) 528 + .all(jsonTyped.target, jsonTyped.collection, field, limit, offset); 529 + 530 + const countResult = db 531 + .prepare(SQL.count) 532 + .get(jsonTyped.target, jsonTyped.collection, field); 533 + const total = countResult ? Number(countResult.total) : 0; 534 + 535 + const linking_records: linksRecord[] = rows.map((row: any) => { 536 + const rkey = row.srcuri.split("/").pop()!; 537 + return { 538 + did: row.srcdid, 539 + collection: row.srccol, 540 + rkey, 541 + }; 542 + }); 543 + 544 + const response: linksRecordsResponse = { 545 + total: total.toString(), 546 + linking_records, 547 + }; 548 + 549 + const nextCursor = offset + linking_records.length; 550 + if (nextCursor < total) { 551 + response.cursor = nextCursor.toString(); 552 + } 553 + 554 + return new Response(JSON.stringify(response), { 555 + headers: withCors({ "Content-Type": "application/json" }), 556 + }); 557 + } 558 + case "/links/distinct-dids": { 559 + const jsonTyped = jsonUntyped as linksQuery; 560 + if (!jsonTyped.collection || !jsonTyped.path) { 561 + return new Response( 562 + JSON.stringify({ 563 + error: "Missing required parameters: collection, path", 564 + }), 565 + { 566 + status: 400, 567 + headers: withCors({ "Content-Type": "application/json" }), 568 + } 569 + ); 570 + } 571 + 572 + const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 573 + /^\./, 574 + "" 575 + )}`; 576 + 577 + const paginatedSql = `${SQL.distinctDids} LIMIT ? OFFSET ?`; 578 + const rows = db 579 + .prepare(paginatedSql) 580 + .all(jsonTyped.target, jsonTyped.collection, field, limit, offset); 581 + 582 + const countResult = db 583 + .prepare(SQL.countDistinctDids) 584 + .get(jsonTyped.target, jsonTyped.collection, field); 585 + const total = countResult ? Number(countResult.total) : 0; 586 + 587 + const linking_dids: string[] = rows.map((row: any) => row.srcdid); 588 + 589 + const response: linksDidsResponse = { 590 + total: total.toString(), 591 + linking_dids, 592 + }; 593 + 594 + const nextCursor = offset + linking_dids.length; 595 + if (nextCursor < total) { 596 + response.cursor = nextCursor.toString(); 597 + } 598 + 599 + return new Response(JSON.stringify(response), { 600 + headers: withCors({ "Content-Type": "application/json" }), 601 + }); 602 + } 603 + case "/links/count": { 604 + const jsonTyped = jsonUntyped as linksQuery; 605 + if (!jsonTyped.collection || !jsonTyped.path) { 606 + return new Response( 607 + JSON.stringify({ 608 + error: "Missing required parameters: collection, path", 609 + }), 610 + { 611 + status: 400, 612 + headers: withCors({ "Content-Type": "application/json" }), 613 + } 614 + ); 615 + } 616 + 617 + const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 618 + /^\./, 619 + "" 620 + )}`; 621 + 622 + const result = db 623 + .prepare(SQL.count) 624 + .get(jsonTyped.target, jsonTyped.collection, field); 625 + 626 + const response: linksCountResponse = { 627 + total: result && result.total ? result.total.toString() : "0", 628 + }; 629 + 630 + return new Response(JSON.stringify(response), { 631 + headers: withCors({ "Content-Type": "application/json" }), 632 + }); 633 + } 634 + case "/links/count/distinct-dids": { 635 + const jsonTyped = jsonUntyped as linksQuery; 636 + if (!jsonTyped.collection || !jsonTyped.path) { 637 + return new Response( 638 + JSON.stringify({ 639 + error: "Missing required parameters: collection, path", 640 + }), 641 + { 642 + status: 400, 643 + headers: withCors({ "Content-Type": "application/json" }), 644 + } 645 + ); 646 + } 647 + 648 + const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 649 + /^\./, 650 + "" 651 + )}`; 652 + 653 + const result = db 654 + .prepare(SQL.countDistinctDids) 655 + .get(jsonTyped.target, jsonTyped.collection, field); 656 + 657 + const response: linksCountResponse = { 658 + total: result && result.total ? result.total.toString() : "0", 659 + }; 660 + 661 + return new Response(JSON.stringify(response), { 662 + headers: withCors({ "Content-Type": "application/json" }), 663 + }); 664 + } 665 + case "/links/all": { 666 + const jsonTyped = jsonUntyped as linksAllQuery; 667 + 668 + const rows = db.prepare(SQL.all).all(jsonTyped.target) as any[]; 669 + 670 + const links: linksAllResponse["links"] = {}; 671 + 672 + for (const row of rows) { 673 + if (!links[row.suburi]) { 674 + links[row.suburi] = {}; 675 + } 676 + links[row.suburi][row.srccol] = { 677 + records: row.records, 678 + distinct_dids: row.distinct_dids, 679 + }; 680 + } 681 + 682 + const response: linksAllResponse = { 683 + links, 684 + }; 685 + 686 + return new Response(JSON.stringify(response), { 687 + headers: withCors({ "Content-Type": "application/json" }), 688 + }); 689 + } 690 + default: { 691 + return new Response( 692 + JSON.stringify({ 693 + error: "NotSupported", 694 + message: 695 + "The requested endpoint is not supported by this Constellation implementation.", 696 + }), 697 + { 698 + status: 404, 699 + headers: withCors({ "Content-Type": "application/json" }), 700 + } 701 + ); 702 + } 703 + } 704 + } 705 + 706 + indexServerIndexer(ctx: indexHandlerContext) { 707 + const record = assertRecord(ctx.value); 708 + //const record = validateRecord(ctx.value); 709 + const db = this.userManager.getDbForDid(ctx.doer); 710 + if (!db) return; 711 + console.log("indexering"); 712 + switch (record?.$type) { 713 + case "app.bsky.feed.like": { 714 + return; 715 + } 716 + case "app.bsky.actor.profile": { 717 + console.log("bsky profuile"); 718 + 719 + try { 720 + const stmt = db.prepare(` 721 + INSERT OR IGNORE INTO app_bsky_actor_profile ( 722 + uri, did, cid, rev, createdat, indexedat, json, 723 + displayname, 724 + description, 725 + avatarcid, 726 + avatarmime, 727 + bannercid, 728 + bannermime 729 + ) VALUES (?, ?, ?, ?, ?, ?, ?, 730 + ?, ?, ?, 731 + ?, ?, ?) 732 + `); 733 + console.log({ 734 + uri: ctx.aturi, 735 + did: ctx.doer, 736 + cid: ctx.cid, 737 + rev: ctx.rev, 738 + createdat: record.createdAt, 739 + indexedat: Date.now(), 740 + json: JSON.stringify(record), 741 + displayname: record.displayName, 742 + description: record.description, 743 + avatarcid: uncid(record.avatar?.ref), 744 + avatarmime: record.avatar?.mimeType, 745 + bannercid: uncid(record.banner?.ref), 746 + bannermime: record.banner?.mimeType, 747 + }); 748 + stmt.run( 749 + ctx.aturi ?? null, 750 + ctx.doer ?? null, 751 + ctx.cid ?? null, 752 + ctx.rev ?? null, 753 + record.createdAt ?? null, 754 + Date.now(), 755 + JSON.stringify(record), 756 + 757 + record.displayName ?? null, 758 + record.description ?? null, 759 + uncid(record.avatar?.ref) ?? null, 760 + record.avatar?.mimeType ?? null, 761 + uncid(record.banner?.ref) ?? null, 762 + record.banner?.mimeType ?? null 763 + // TODO please add pinned posts 764 + ); 765 + } catch (err) { 766 + console.error("stmt.run failed:", err); 767 + } 768 + return; 769 + } 770 + case "app.bsky.feed.post": { 771 + console.log("bsky post"); 772 + const stmt = db.prepare(` 773 + INSERT OR IGNORE INTO app_bsky_feed_post ( 774 + uri, did, cid, rev, createdat, indexedat, json, 775 + text, replyroot, replyparent, quote, 776 + imagecount, image1cid, image1mime, image1aspect, 777 + image2cid, image2mime, image2aspect, 778 + image3cid, image3mime, image3aspect, 779 + image4cid, image4mime, image4aspect, 780 + videocount, videocid, videomime, videoaspect 781 + ) VALUES (?, ?, ?, ?, ?, ?, ?, 782 + ?, ?, ?, ?, 783 + ?, ?, ?, ?, 784 + ?, ?, ?, 785 + ?, ?, ?, 786 + ?, ?, ?, 787 + ?, ?, ?, ?) 788 + `); 789 + 790 + const embed = record.embed; 791 + 792 + const images = extractImages(embed); 793 + const video = extractVideo(embed); 794 + const quoteUri = extractQuoteUri(embed); 795 + try { 796 + stmt.run( 797 + ctx.aturi ?? null, 798 + ctx.doer ?? null, 799 + ctx.cid ?? null, 800 + ctx.rev ?? null, 801 + record.createdAt, 802 + Date.now(), 803 + JSON.stringify(record), 804 + 805 + record.text ?? null, 806 + record.reply?.root?.uri ?? null, 807 + record.reply?.parent?.uri ?? null, 808 + 809 + quoteUri, 810 + 811 + images.length, 812 + uncid(images[0]?.image?.ref) ?? null, 813 + images[0]?.image?.mimeType ?? null, 814 + images[0]?.aspectRatio && 815 + images[0].aspectRatio.width && 816 + images[0].aspectRatio.height 817 + ? `${images[0].aspectRatio.width}:${images[0].aspectRatio.height}` 818 + : null, 819 + 820 + uncid(images[1]?.image?.ref) ?? null, 821 + images[1]?.image?.mimeType ?? null, 822 + images[1]?.aspectRatio && 823 + images[1].aspectRatio.width && 824 + images[1].aspectRatio.height 825 + ? `${images[1].aspectRatio.width}:${images[1].aspectRatio.height}` 826 + : null, 827 + 828 + uncid(images[2]?.image?.ref) ?? null, 829 + images[2]?.image?.mimeType ?? null, 830 + images[2]?.aspectRatio && 831 + images[2].aspectRatio.width && 832 + images[2].aspectRatio.height 833 + ? `${images[2].aspectRatio.width}:${images[2].aspectRatio.height}` 834 + : null, 835 + 836 + uncid(images[3]?.image?.ref) ?? null, 837 + images[3]?.image?.mimeType ?? null, 838 + images[3]?.aspectRatio && 839 + images[3].aspectRatio.width && 840 + images[3].aspectRatio.height 841 + ? `${images[3].aspectRatio.width}:${images[3].aspectRatio.height}` 842 + : null, 843 + 844 + uncid(video?.video) ? 1 : 0, 845 + uncid(video?.video) ?? null, 846 + uncid(video?.video) ? "video/mp4" : null, 847 + video?.aspectRatio 848 + ? `${video.aspectRatio.width}:${video.aspectRatio.height}` 849 + : null 850 + ); 851 + } catch (err) { 852 + console.error("stmt.run failed:", err); 853 + } 854 + return; 855 + } 856 + default: { 857 + // what the hell 858 + return; 859 + } 860 + } 861 + } 862 + 863 + // user data 864 + queryProfileView( 865 + did: string, 866 + type: "" 867 + ): ATPAPI.AppBskyActorDefs.ProfileView | undefined; 868 + queryProfileView( 869 + did: string, 870 + type: "Basic" 871 + ): ATPAPI.AppBskyActorDefs.ProfileViewBasic | undefined; 872 + queryProfileView( 873 + did: string, 874 + type: "Detailed" 875 + ): ATPAPI.AppBskyActorDefs.ProfileViewDetailed | undefined; 876 + queryProfileView( 877 + did: string, 878 + type: "" | "Basic" | "Detailed" 879 + ): 880 + | ATPAPI.AppBskyActorDefs.ProfileView 881 + | ATPAPI.AppBskyActorDefs.ProfileViewBasic 882 + | ATPAPI.AppBskyActorDefs.ProfileViewDetailed 883 + | undefined { 884 + if (!this.isRegisteredIndexUser(did)) return; 885 + const db = this.userManager.getDbForDid(did); 886 + if (!db) return; 887 + 888 + const stmt = db.prepare(` 889 + SELECT * 890 + FROM app_bsky_actor_profile 891 + WHERE did = ? 892 + LIMIT 1; 893 + `); 894 + 895 + const row = stmt.get(did) as ProfileRow; 896 + 897 + // simulate different types returned 898 + switch (type) { 899 + case "": { 900 + const result: ATPAPI.AppBskyActorDefs.ProfileView = { 901 + $type: "app.bsky.actor.defs#profileView", 902 + did: did, 903 + handle: "idiot.fuck.shit.example.com", // TODO: Resolve user identity here for the handle 904 + displayName: row.displayname ?? undefined, 905 + description: row.description ?? undefined, 906 + avatar: "https://google.com/", // create profile URL from resolved identity 907 + //associated?: ProfileAssociated, 908 + indexedAt: row.createdat 909 + ? new Date(row.createdat).toISOString() 910 + : undefined, 911 + createdAt: row.createdat 912 + ? new Date(row.createdat).toISOString() 913 + : undefined, 914 + //viewer?: ViewerState, 915 + //labels?: ComAtprotoLabelDefs.Label[], 916 + //verification?: VerificationState, 917 + //status?: StatusView, 918 + }; 919 + return result; 920 + } 921 + case "Basic": { 922 + const result: ATPAPI.AppBskyActorDefs.ProfileViewBasic = { 923 + $type: "app.bsky.actor.defs#profileViewBasic", 924 + did: did, 925 + handle: "idiot.fuck.shit.example.com", // TODO: Resolve user identity here for the handle 926 + displayName: row.displayname ?? undefined, 927 + avatar: "https://google.com/", // create profile URL from resolved identity 928 + //associated?: ProfileAssociated, 929 + createdAt: row.createdat 930 + ? new Date(row.createdat).toISOString() 931 + : undefined, 932 + //viewer?: ViewerState, 933 + //labels?: ComAtprotoLabelDefs.Label[], 934 + //verification?: VerificationState, 935 + //status?: StatusView, 936 + }; 937 + return result; 938 + } 939 + case "Detailed": { 940 + // Query for follower count from the backlink_skeleton table 941 + const followersStmt = db.prepare(` 942 + SELECT COUNT(*) as count 943 + FROM backlink_skeleton 944 + WHERE subdid = ? AND srccol = 'app.bsky.graph.follow' 945 + `); 946 + const followersResult = followersStmt.get(did) as { count: number }; 947 + const followersCount = followersResult?.count ?? 0; 948 + 949 + // Query for following count from the app_bsky_graph_follow table 950 + const followingStmt = db.prepare(` 951 + SELECT COUNT(*) as count 952 + FROM app_bsky_graph_follow 953 + WHERE did = ? 954 + `); 955 + const followingResult = followingStmt.get(did) as { count: number }; 956 + const followsCount = followingResult?.count ?? 0; 957 + 958 + // Query for post count from the app_bsky_feed_post table 959 + const postsStmt = db.prepare(` 960 + SELECT COUNT(*) as count 961 + FROM app_bsky_feed_post 962 + WHERE did = ? 963 + `); 964 + const postsResult = postsStmt.get(did) as { count: number }; 965 + const postsCount = postsResult?.count ?? 0; 966 + 967 + const result: ATPAPI.AppBskyActorDefs.ProfileViewDetailed = { 968 + $type: "app.bsky.actor.defs#profileViewDetailed", 969 + did: did, 970 + handle: "idiot.fuck.shit.example.com", // TODO: Resolve user identity here for the handle 971 + displayName: row.displayname ?? undefined, 972 + description: row.description ?? undefined, 973 + avatar: "https://google.com/", // TODO: create profile URL from resolved identity 974 + banner: "https://youtube.com/", // same here 975 + followersCount: followersCount, 976 + followsCount: followsCount, 977 + postsCount: postsCount, 978 + //associated?: ProfileAssociated, 979 + //joinedViaStarterPack?: // AppBskyGraphDefs.StarterPackViewBasic; 980 + indexedAt: row.createdat 981 + ? new Date(row.createdat).toISOString() 982 + : undefined, 983 + createdAt: row.createdat 984 + ? new Date(row.createdat).toISOString() 985 + : undefined, 986 + //viewer?: ViewerState, 987 + //labels?: ComAtprotoLabelDefs.Label[], 988 + pinnedPost: undefined, //row.; // TODO: i forgot to put pinnedp posts in db schema oops 989 + //verification?: VerificationState, 990 + //status?: StatusView, 991 + }; 992 + return result; 993 + } 994 + default: 995 + throw new Error("Invalid type"); 996 + } 997 + } 998 + 999 + // post hydration 1000 + queryPostView(uri: string): ATPAPI.AppBskyFeedDefs.PostView | undefined { 1001 + const URI = new AtUri(uri); 1002 + const did = URI.host; 1003 + if (!this.isRegisteredIndexUser(did)) return; 1004 + const db = this.userManager.getDbForDid(did); 1005 + if (!db) return; 1006 + 1007 + const stmt = db.prepare(` 1008 + SELECT * 1009 + FROM app_bsky_feed_post 1010 + WHERE uri = ? 1011 + LIMIT 1; 1012 + `); 1013 + 1014 + const row = stmt.get(uri) as PostRow; 1015 + const profileView = this.queryProfileView(did, "Basic"); 1016 + if (!row || !row.cid || !profileView || !row.json) return; 1017 + const value = JSON.parse(row.json) as ATPAPI.AppBskyFeedPost.Record; 1018 + 1019 + const post: ATPAPI.AppBskyFeedDefs.PostView = { 1020 + uri: row.uri, 1021 + cid: row.cid, 1022 + author: profileView, 1023 + record: value, 1024 + indexedAt: new Date(row.indexedat).toISOString(), 1025 + embed: value.embed, 1026 + }; 1027 + 1028 + return post; 1029 + } 1030 + queryFeedViewPost( 1031 + uri: string 1032 + ): ATPAPI.AppBskyFeedDefs.FeedViewPost | undefined { 1033 + const post = this.queryPostView(uri); 1034 + if (!post) return; 1035 + 1036 + const feedviewpost: ATPAPI.AppBskyFeedDefs.FeedViewPost = { 1037 + $type: "app.bsky.feed.defs#feedViewPost", 1038 + post: post, 1039 + //reply: ReplyRef, 1040 + //reason: , 1041 + }; 1042 + 1043 + return feedviewpost; 1044 + } 1045 + 1046 + // user feedgens 1047 + 1048 + queryActorFeeds(did: string): ATPAPI.AppBskyFeedDefs.GeneratorView[] { 1049 + if (!this.isRegisteredIndexUser(did)) return []; 1050 + const db = this.userManager.getDbForDid(did); 1051 + if (!db) return []; 1052 + 1053 + const stmt = db.prepare(` 1054 + SELECT uri, cid, did, json, indexedat 1055 + FROM app_bsky_feed_generator 1056 + WHERE did = ? 1057 + ORDER BY createdat DESC; 1058 + `); 1059 + 1060 + const rows = stmt.all(did) as unknown as GeneratorRow[]; 1061 + const creatorView = this.queryProfileView(did, "Basic"); 1062 + if (!creatorView) return []; 1063 + 1064 + return rows 1065 + .map((row) => { 1066 + try { 1067 + if (!row.json) return; 1068 + const record = JSON.parse( 1069 + row.json 1070 + ) as ATPAPI.AppBskyFeedGenerator.Record; 1071 + return { 1072 + $type: "app.bsky.feed.defs#generatorView", 1073 + uri: row.uri, 1074 + cid: row.cid, 1075 + did: row.did, 1076 + creator: creatorView, 1077 + displayName: record.displayName, 1078 + description: record.description, 1079 + descriptionFacets: record.descriptionFacets, 1080 + avatar: record.avatar, 1081 + likeCount: 0, // TODO: this should be easy 1082 + indexedAt: new Date(row.indexedat).toISOString(), 1083 + } as ATPAPI.AppBskyFeedDefs.GeneratorView; 1084 + } catch { 1085 + return undefined; 1086 + } 1087 + }) 1088 + .filter((v): v is ATPAPI.AppBskyFeedDefs.GeneratorView => !!v); 1089 + } 1090 + 1091 + queryFeedGenerator( 1092 + uri: string 1093 + ): ATPAPI.AppBskyFeedDefs.GeneratorView | undefined { 1094 + return this.queryFeedGenerators([uri])[0]; 1095 + } 1096 + 1097 + queryFeedGenerators(uris: string[]): ATPAPI.AppBskyFeedDefs.GeneratorView[] { 1098 + const generators: ATPAPI.AppBskyFeedDefs.GeneratorView[] = []; 1099 + const urisByDid = new Map<string, string[]>(); 1100 + 1101 + for (const uri of uris) { 1102 + try { 1103 + const { host: did } = new AtUri(uri); 1104 + if (!urisByDid.has(did)) { 1105 + urisByDid.set(did, []); 1106 + } 1107 + urisByDid.get(did)!.push(uri); 1108 + } catch {} 1109 + } 1110 + 1111 + for (const [did, didUris] of urisByDid.entries()) { 1112 + if (!this.isRegisteredIndexUser(did)) continue; 1113 + const db = this.userManager.getDbForDid(did); 1114 + if (!db) continue; 1115 + 1116 + const placeholders = didUris.map(() => "?").join(","); 1117 + const stmt = db.prepare(` 1118 + SELECT uri, cid, did, json, indexedat 1119 + FROM app_bsky_feed_generator 1120 + WHERE uri IN (${placeholders}); 1121 + `); 1122 + 1123 + const rows = stmt.all(...didUris) as unknown as GeneratorRow[]; 1124 + if (rows.length === 0) continue; 1125 + 1126 + const creatorView = this.queryProfileView(did, ""); 1127 + if (!creatorView) continue; 1128 + 1129 + for (const row of rows) { 1130 + try { 1131 + if (!row.json || !row.cid) continue; 1132 + const record = JSON.parse( 1133 + row.json 1134 + ) as ATPAPI.AppBskyFeedGenerator.Record; 1135 + generators.push({ 1136 + $type: "app.bsky.feed.defs#generatorView", 1137 + uri: row.uri, 1138 + cid: row.cid, 1139 + did: row.did, 1140 + creator: creatorView, 1141 + displayName: record.displayName, 1142 + description: record.description, 1143 + descriptionFacets: record.descriptionFacets, 1144 + avatar: record.avatar as string | undefined, 1145 + likeCount: 0, 1146 + indexedAt: new Date(row.indexedat).toISOString(), 1147 + }); 1148 + } catch {} 1149 + } 1150 + } 1151 + return generators; 1152 + } 1153 + 1154 + // user feeds 1155 + 1156 + queryAuthorFeed( 1157 + did: string, 1158 + cursor?: string 1159 + ): 1160 + | { 1161 + items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1162 + cursor: string | undefined; 1163 + } 1164 + | undefined { 1165 + if (!this.isRegisteredIndexUser(did)) return; 1166 + const db = this.userManager.getDbForDid(did); 1167 + if (!db) return; 1168 + 1169 + // TODO: implement this for real 1170 + let query = ` 1171 + SELECT uri, indexedat, cid 1172 + FROM app_bsky_feed_post 1173 + WHERE did = ? 1174 + `; 1175 + const params: (string | number)[] = [did]; 1176 + 1177 + if (cursor) { 1178 + const [indexedat, cid] = cursor.split("::"); 1179 + query += ` AND (indexedat < ? OR (indexedat = ? AND cid < ?))`; 1180 + params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); 1181 + } 1182 + 1183 + query += ` ORDER BY indexedat DESC, cid DESC LIMIT ${FEED_LIMIT}`; 1184 + 1185 + const stmt = db.prepare(query); 1186 + const rows = stmt.all(...params) as { 1187 + uri: string; 1188 + indexedat: number; 1189 + cid: string; 1190 + }[]; 1191 + 1192 + const items = rows 1193 + .map((row) => this.queryFeedViewPost(row.uri)) // TODO: for replies and repost i should inject the reason here 1194 + .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1195 + 1196 + const lastItem = rows[rows.length - 1]; 1197 + const nextCursor = lastItem 1198 + ? `${lastItem.indexedat}::${lastItem.cid}` 1199 + : undefined; 1200 + 1201 + return { items, cursor: nextCursor }; 1202 + } 1203 + 1204 + queryListFeed( 1205 + uri: string, 1206 + cursor?: string 1207 + ): 1208 + | { 1209 + items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1210 + cursor: string | undefined; 1211 + } 1212 + | undefined { 1213 + return { items: [], cursor: undefined }; 1214 + } 1215 + 1216 + queryActorLikes( 1217 + did: string, 1218 + cursor?: string 1219 + ): 1220 + | { 1221 + items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1222 + cursor: string | undefined; 1223 + } 1224 + | undefined { 1225 + if (!this.isRegisteredIndexUser(did)) return; 1226 + const db = this.userManager.getDbForDid(did); 1227 + if (!db) return; 1228 + 1229 + let query = ` 1230 + SELECT subject, indexedat, cid 1231 + FROM app_bsky_feed_like 1232 + WHERE did = ? 1233 + `; 1234 + const params: (string | number)[] = [did]; 1235 + 1236 + if (cursor) { 1237 + const [indexedat, cid] = cursor.split("::"); 1238 + query += ` AND (indexedat < ? OR (indexedat = ? AND cid < ?))`; 1239 + params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); 1240 + } 1241 + 1242 + query += ` ORDER BY indexedat DESC, cid DESC LIMIT ${FEED_LIMIT}`; 1243 + 1244 + const stmt = db.prepare(query); 1245 + const rows = stmt.all(...params) as { 1246 + subject: string; 1247 + indexedat: number; 1248 + cid: string; 1249 + }[]; 1250 + 1251 + const items = rows 1252 + .map((row) => this.queryFeedViewPost(row.subject)) 1253 + .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1254 + 1255 + const lastItem = rows[rows.length - 1]; 1256 + const nextCursor = lastItem 1257 + ? `${lastItem.indexedat}::${lastItem.cid}` 1258 + : undefined; 1259 + 1260 + return { items, cursor: nextCursor }; 1261 + } 1262 + 1263 + // post metadata 1264 + 1265 + queryLikes(uri: string): ATPAPI.AppBskyFeedGetLikes.Like[] | undefined { 1266 + const postUri = new AtUri(uri); 1267 + const postAuthorDid = postUri.hostname; 1268 + if (!this.isRegisteredIndexUser(postAuthorDid)) return; 1269 + const db = this.userManager.getDbForDid(postAuthorDid); 1270 + if (!db) return; 1271 + 1272 + const stmt = db.prepare(` 1273 + SELECT b.srcdid, b.srcuri 1274 + FROM backlink_skeleton AS b 1275 + WHERE b.suburi = ? AND b.srccol = 'app_bsky_feed_like' 1276 + ORDER BY b.id DESC; 1277 + `); 1278 + 1279 + const rows = stmt.all(uri) as unknown as BacklinkRow[]; 1280 + 1281 + return rows 1282 + .map((row) => { 1283 + const actor = this.queryProfileView(row.srcdid, ""); 1284 + if (!actor) return; 1285 + 1286 + return { 1287 + // TODO write indexedAt for spacedust indexes 1288 + createdAt: new Date(Date.now()).toISOString(), 1289 + indexedAt: new Date(Date.now()).toISOString(), 1290 + actor: actor, 1291 + }; 1292 + }) 1293 + .filter((like): like is ATPAPI.AppBskyFeedGetLikes.Like => !!like); 1294 + } 1295 + 1296 + queryReposts(uri: string): ATPAPI.AppBskyActorDefs.ProfileView[] { 1297 + const postUri = new AtUri(uri); 1298 + const postAuthorDid = postUri.hostname; 1299 + if (!this.isRegisteredIndexUser(postAuthorDid)) return []; 1300 + const db = this.userManager.getDbForDid(postAuthorDid); 1301 + if (!db) return []; 1302 + 1303 + const stmt = db.prepare(` 1304 + SELECT srcdid 1305 + FROM backlink_skeleton 1306 + WHERE suburi = ? AND srccol = 'app_bsky_feed_repost' 1307 + ORDER BY id DESC; 1308 + `); 1309 + 1310 + const rows = stmt.all(uri) as { srcdid: string }[]; 1311 + 1312 + return rows 1313 + .map((row) => this.queryProfileView(row.srcdid, "")) 1314 + .filter((p): p is ATPAPI.AppBskyActorDefs.ProfileView => !!p); 1315 + } 1316 + 1317 + queryQuotes(uri: string): ATPAPI.AppBskyFeedDefs.FeedViewPost[] { 1318 + const postUri = new AtUri(uri); 1319 + const postAuthorDid = postUri.hostname; 1320 + if (!this.isRegisteredIndexUser(postAuthorDid)) return []; 1321 + const db = this.userManager.getDbForDid(postAuthorDid); 1322 + if (!db) return []; 1323 + 1324 + const stmt = db.prepare(` 1325 + SELECT srcuri 1326 + FROM backlink_skeleton 1327 + WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'quote' 1328 + ORDER BY id DESC; 1329 + `); 1330 + 1331 + const rows = stmt.all(uri) as { srcuri: string }[]; 1332 + 1333 + return rows 1334 + .map((row) => this.queryFeedViewPost(row.srcuri)) 1335 + .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1336 + } 1337 + 1338 + queryPostThread( 1339 + uri: string 1340 + ): ATPAPI.AppBskyFeedGetPostThread.OutputSchema | undefined { 1341 + const post = this.queryPostView(uri); 1342 + if (!post) { 1343 + return { 1344 + thread: { 1345 + $type: "app.bsky.feed.defs#notFoundPost", 1346 + uri: uri, 1347 + notFound: true, 1348 + } as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.NotFoundPost>, 1349 + }; 1350 + } 1351 + 1352 + const thread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1353 + $type: "app.bsky.feed.defs#threadViewPost", 1354 + post: post, 1355 + replies: [], 1356 + }; 1357 + 1358 + let current = thread; 1359 + while ((current.post.record.reply as any)?.parent?.uri) { 1360 + const parentUri = (current.post.record.reply as any)?.parent?.uri; 1361 + const parentPost = this.queryPostView(parentUri); 1362 + if (!parentPost) break; 1363 + 1364 + const parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1365 + $type: "app.bsky.feed.defs#threadViewPost", 1366 + post: parentPost, 1367 + replies: [ 1368 + current as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>, 1369 + ], 1370 + }; 1371 + current.parent = 1372 + parentThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>; 1373 + current = parentThread; 1374 + } 1375 + 1376 + const seenUris = new Set<string>(); 1377 + const fetchReplies = ( 1378 + parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost 1379 + ) => { 1380 + if (seenUris.has(parentThread.post.uri)) return; 1381 + seenUris.add(parentThread.post.uri); 1382 + 1383 + const parentUri = new AtUri(parentThread.post.uri); 1384 + const parentAuthorDid = parentUri.hostname; 1385 + const db = this.userManager.getDbForDid(parentAuthorDid); 1386 + if (!db) return; 1387 + 1388 + const stmt = db.prepare(` 1389 + SELECT srcuri 1390 + FROM backlink_skeleton 1391 + WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'replyparent' 1392 + `); 1393 + const replyRows = stmt.all(parentThread.post.uri) as { srcuri: string }[]; 1394 + 1395 + const replies = replyRows 1396 + .map((row) => this.queryPostView(row.srcuri)) 1397 + .filter((p): p is ATPAPI.AppBskyFeedDefs.PostView => !!p); 1398 + 1399 + for (const replyPost of replies) { 1400 + const replyThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1401 + $type: "app.bsky.feed.defs#threadViewPost", 1402 + post: replyPost, 1403 + parent: 1404 + parentThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>, 1405 + replies: [], 1406 + }; 1407 + parentThread.replies?.push( 1408 + replyThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost> 1409 + ); 1410 + fetchReplies(replyThread); 1411 + } 1412 + }; 1413 + 1414 + fetchReplies(thread); 1415 + 1416 + const returned = 1417 + thread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>; 1418 + 1419 + return { thread: returned }; 1420 + } 1421 + 1422 + /** 1423 + * please do not use this, use openDbForDid() instead 1424 + * @param did 1425 + * @returns 1426 + */ 1427 + internalCreateDbForDid(did: string): Database { 1428 + const path = `${this.config.baseDbPath}/${did}.sqlite`; 1429 + const db = new Database(path); 1430 + setupUserDb(db); 1431 + //await db.exec(/* CREATE IF NOT EXISTS statements */); 1432 + return db; 1433 + } 1434 + 1435 + isRegisteredIndexUser(did: string): boolean { 1436 + const stmt = this.systemDB.prepare(` 1437 + SELECT 1 1438 + FROM users 1439 + WHERE did = ? 1440 + AND onboardingstatus != 'onboarding-backfill' 1441 + LIMIT 1; 1442 + `); 1443 + const result = stmt.value<[number]>(did); 1444 + const exists = result !== undefined; 1445 + return exists; 1446 + } 1447 + } 1448 + 16 1449 export class IndexServerUserManager { 1450 + public indexServer: IndexServer; 1451 + 1452 + constructor(indexServer: IndexServer) { 1453 + this.indexServer = indexServer; 1454 + } 1455 + 17 1456 private users = new Map<string, UserIndexServer>(); 18 1457 19 1458 /*async*/ addUser(did: string) { 20 1459 if (this.users.has(did)) return; 21 - const instance = new UserIndexServer(did); 1460 + const instance = new UserIndexServer(this, did); 22 1461 //await instance.initialize(); 23 1462 this.users.set(did, instance); 24 1463 } ··· 60 1499 } 61 1500 62 1501 class UserIndexServer { 1502 + public indexServerUserManager: IndexServerUserManager; 63 1503 did: string; 64 1504 db: Database; // | undefined; 65 1505 jetstream: JetstreamManager; // | undefined; 66 1506 spacedust: SpacedustManager; // | undefined; 67 1507 68 - constructor(did: string) { 1508 + constructor(indexServerUserManager: IndexServerUserManager, did: string) { 69 1509 this.did = did; 70 - this.db = internalCreateDbForDid(this.did); 1510 + this.indexServerUserManager = indexServerUserManager; 1511 + this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid(this.did); 71 1512 // should probably put the params of exactly what were listening to here 72 1513 this.jetstream = new JetstreamManager((msg) => { 73 1514 console.log("Received Jetstream message: ", msg); ··· 79 1520 const value = msg.commit.record; 80 1521 81 1522 if (!doer || !value) return; 82 - indexServerIndexer({ 1523 + this.indexServerUserManager.indexServer.indexServerIndexer({ 83 1524 op, 84 1525 doer, 85 1526 cid: msg.commit.cid, ··· 227 1668 } 228 1669 } 229 1670 230 - function isRegisteredIndexUser(did: string): boolean { 231 - const stmt = systemDB.prepare(` 232 - SELECT 1 233 - FROM users 234 - WHERE did = ? 235 - AND onboardingstatus != 'onboarding-backfill' 236 - LIMIT 1; 237 - `); 238 - const result = stmt.value<[number]>(did); 239 - const exists = result !== undefined; 240 - return exists; 241 - } 242 - 243 - /** 244 - * please do not use this, use openDbForDid() instead 245 - * @param did 246 - * @returns 247 - */ 248 - function internalCreateDbForDid(did: string): Database { 249 - const path = `./dbs/${did}.sqlite`; 250 - const db = new Database(path); 251 - setupUserDb(db); 252 - //await db.exec(/* CREATE IF NOT EXISTS statements */); 253 - return db; 254 - } 1671 + // /** 1672 + // * please do not use this, use openDbForDid() instead 1673 + // * @param did 1674 + // * @returns 1675 + // */ 1676 + // function internalCreateDbForDid(did: string): Database { 1677 + // const path = `./dbs/${did}.sqlite`; 1678 + // const db = new Database(path); 1679 + // setupUserDb(db); 1680 + // //await db.exec(/* CREATE IF NOT EXISTS statements */); 1681 + // return db; 1682 + // } 255 1683 256 - function getDbForDid(did: string): Database | undefined { 257 - const db = indexerUserManager.getDbForDid(did); 258 - if (!db) return; 259 - return db; 260 - } 1684 + // function getDbForDid(did: string): Database | undefined { 1685 + // const db = indexerUserManager.getDbForDid(did); 1686 + // if (!db) return; 1687 + // return db; 1688 + // } 261 1689 262 1690 // async function connectToJetstream(did: string, db: Database): Promise<WebSocket> { 263 1691 // const url = `${jetstreamurl}/xrpc/com.atproto.sync.subscribeRepos?did=${did}`; ··· 342 1770 bannermime: string | null; 343 1771 }; 344 1772 345 - export async function indexServerHandler(req: Request): Promise<Response> { 346 - const url = new URL(req.url); 347 - const pathname = url.pathname; 348 - //const bskyUrl = `https://api.bsky.app${pathname}${url.search}`; 349 - //const hasAuth = req.headers.has("authorization"); 350 - const xrpcMethod = pathname.startsWith("/xrpc/") 351 - ? pathname.slice("/xrpc/".length) 352 - : null; 353 - const searchParams = searchParamsToJson(url.searchParams); 354 - console.log(JSON.stringify(searchParams, null, 2)); 355 - const jsonUntyped = searchParams; 356 - 357 - switch (xrpcMethod) { 358 - case "app.bsky.actor.getProfile": { 359 - const jsonTyped = 360 - jsonUntyped as IndexServerTypes.AppBskyActorGetProfile.QueryParams; 361 - 362 - const res = queryProfileView(jsonTyped.actor, "Detailed"); 363 - if (!res) 364 - return new Response( 365 - JSON.stringify({ 366 - error: "User not found", 367 - }), 368 - { 369 - status: 404, 370 - headers: withCors({ "Content-Type": "application/json" }), 371 - } 372 - ); 373 - const response: IndexServerTypes.AppBskyActorGetProfile.OutputSchema = 374 - res; 375 - 376 - return new Response(JSON.stringify(response), { 377 - headers: withCors({ "Content-Type": "application/json" }), 378 - }); 379 - } 380 - case "app.bsky.actor.getProfiles": { 381 - const jsonTyped = 382 - jsonUntyped as IndexServerTypes.AppBskyActorGetProfiles.QueryParams; 383 - 384 - if (typeof jsonUntyped?.actors === "string" ) { 385 - const res = queryProfileView(jsonUntyped.actors as string, "Detailed"); 386 - if (!res) 387 - return new Response( 388 - JSON.stringify({ 389 - error: "User not found", 390 - }), 391 - { 392 - status: 404, 393 - headers: withCors({ "Content-Type": "application/json" }), 394 - } 395 - ); 396 - const response: IndexServerTypes.AppBskyActorGetProfiles.OutputSchema = { 397 - profiles: [res], 398 - }; 399 - 400 - return new Response(JSON.stringify(response), { 401 - headers: withCors({ "Content-Type": "application/json" }), 402 - }); 403 - } 404 - 405 - const res: ATPAPI.AppBskyActorDefs.ProfileViewDetailed[] = 406 - jsonTyped.actors 407 - .map((actor) => { 408 - return queryProfileView(actor, "Detailed"); 409 - }) 410 - .filter( 411 - (x): x is ATPAPI.AppBskyActorDefs.ProfileViewDetailed => 412 - x !== undefined 413 - ); 414 - 415 - if (!res) 416 - return new Response( 417 - JSON.stringify({ 418 - error: "User not found", 419 - }), 420 - { 421 - status: 404, 422 - headers: withCors({ "Content-Type": "application/json" }), 423 - } 424 - ); 425 - 426 - const response: IndexServerTypes.AppBskyActorGetProfiles.OutputSchema = { 427 - profiles: res, 428 - }; 429 - 430 - return new Response(JSON.stringify(response), { 431 - headers: withCors({ "Content-Type": "application/json" }), 432 - }); 433 - } 434 - case "app.bsky.feed.getActorFeeds": { 435 - const jsonTyped = 436 - jsonUntyped as IndexServerTypes.AppBskyFeedGetActorFeeds.QueryParams; 437 - 438 - const qresult = queryActorFeeds(jsonTyped.actor) 439 - 440 - const response: IndexServerTypes.AppBskyFeedGetActorFeeds.OutputSchema = 441 - { 442 - feeds: qresult 443 - }; 444 - 445 - return new Response(JSON.stringify(response), { 446 - headers: withCors({ "Content-Type": "application/json" }), 447 - }); 448 - } 449 - case "app.bsky.feed.getFeedGenerator": { 450 - const jsonTyped = 451 - jsonUntyped as IndexServerTypes.AppBskyFeedGetFeedGenerator.QueryParams; 452 - 453 - const qresult = queryFeedGenerator(jsonTyped.feed) 454 - if (!qresult) { 455 - return new Response( 456 - JSON.stringify({ 457 - error: "Feed not found", 458 - }), 459 - { 460 - status: 404, 461 - headers: withCors({ "Content-Type": "application/json" }), 462 - } 463 - ); 464 - } 465 - 466 - const response: IndexServerTypes.AppBskyFeedGetFeedGenerator.OutputSchema = 467 - { 468 - view: qresult, 469 - isOnline: true, // lmao 470 - isValid: true, // lmao 471 - }; 472 - 473 - return new Response(JSON.stringify(response), { 474 - headers: withCors({ "Content-Type": "application/json" }), 475 - }); 476 - } 477 - case "app.bsky.feed.getFeedGenerators": { 478 - const jsonTyped = 479 - jsonUntyped as IndexServerTypes.AppBskyFeedGetFeedGenerators.QueryParams; 480 - 481 - const qresult = queryFeedGenerators(jsonTyped.feeds) 482 - if (!qresult) { 483 - return new Response( 484 - JSON.stringify({ 485 - error: "Feed not found", 486 - }), 487 - { 488 - status: 404, 489 - headers: withCors({ "Content-Type": "application/json" }), 490 - } 491 - ); 492 - } 493 - 494 - const response: IndexServerTypes.AppBskyFeedGetFeedGenerators.OutputSchema = 495 - { 496 - feeds: qresult 497 - }; 498 - 499 - return new Response(JSON.stringify(response), { 500 - headers: withCors({ "Content-Type": "application/json" }), 501 - }); 502 - } 503 - case "app.bsky.feed.getPosts": { 504 - const jsonTyped = 505 - jsonUntyped as IndexServerTypes.AppBskyFeedGetPosts.QueryParams; 506 - 507 - const posts: IndexServerTypes.AppBskyFeedGetPosts.OutputSchema["posts"] = 508 - jsonTyped.uris 509 - .map((uri) => { 510 - return queryPostView(uri); 511 - }) 512 - .filter(Boolean) as ATPAPI.AppBskyFeedDefs.PostView[]; 513 - 514 - const response: IndexServerTypes.AppBskyFeedGetPosts.OutputSchema = { 515 - posts, 516 - }; 517 - 518 - return new Response(JSON.stringify(response), { 519 - headers: withCors({ "Content-Type": "application/json" }), 520 - }); 521 - } 522 - case "party.whey.app.bsky.feed.getActorLikesPartial": { 523 - const jsonTyped = 524 - jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetActorLikesPartial.QueryParams; 525 - 526 - // TODO: not partial yet, currently skips refs 527 - 528 - const qresult = queryActorLikes(jsonTyped.actor, jsonTyped.cursor) 529 - if (!qresult) { 530 - return new Response( 531 - JSON.stringify({ 532 - error: "Feed not found", 533 - }), 534 - { 535 - status: 404, 536 - headers: withCors({ "Content-Type": "application/json" }), 537 - } 538 - ); 539 - } 540 - 541 - const response: IndexServerTypes.PartyWheyAppBskyFeedGetActorLikesPartial.OutputSchema = 542 - { 543 - feed: qresult.items as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.FeedViewPost>[], 544 - cursor: qresult.cursor 545 - }; 546 - 547 - return new Response(JSON.stringify(response), { 548 - headers: withCors({ "Content-Type": "application/json" }), 549 - }); 550 - } 551 - case "party.whey.app.bsky.feed.getAuthorFeedPartial": { 552 - const jsonTyped = 553 - jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetAuthorFeedPartial.QueryParams; 554 - 555 - // TODO: not partial yet, currently skips refs 556 - 557 - const qresult = queryAuthorFeed(jsonTyped.actor, jsonTyped.cursor) 558 - if (!qresult) { 559 - return new Response( 560 - JSON.stringify({ 561 - error: "Feed not found", 562 - }), 563 - { 564 - status: 404, 565 - headers: withCors({ "Content-Type": "application/json" }), 566 - } 567 - ); 568 - } 569 - 570 - const response: IndexServerTypes.PartyWheyAppBskyFeedGetAuthorFeedPartial.OutputSchema = 571 - { 572 - feed: qresult.items as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.FeedViewPost>[], 573 - cursor: qresult.cursor 574 - }; 575 - 576 - return new Response(JSON.stringify(response), { 577 - headers: withCors({ "Content-Type": "application/json" }), 578 - }); 579 - } 580 - case "party.whey.app.bsky.feed.getLikesPartial": { 581 - const jsonTyped = 582 - jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetLikesPartial.QueryParams; 583 - 584 - // TODO: not partial yet, currently skips refs 585 - 586 - const qresult = queryLikes(jsonTyped.uri) 587 - if (!qresult) { 588 - return new Response( 589 - JSON.stringify({ 590 - error: "Feed not found", 591 - }), 592 - { 593 - status: 404, 594 - headers: withCors({ "Content-Type": "application/json" }), 595 - } 596 - ); 597 - } 598 - const response: IndexServerTypes.PartyWheyAppBskyFeedGetLikesPartial.OutputSchema = 599 - { 600 - // @ts-ignore whatever i dont care TODO: fix ts ignores 601 - likes: qresult 602 - }; 603 - 604 - return new Response(JSON.stringify(response), { 605 - headers: withCors({ "Content-Type": "application/json" }), 606 - }); 607 - } 608 - case "party.whey.app.bsky.feed.getPostThreadPartial": { 609 - const jsonTyped = 610 - jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.QueryParams; 611 - 612 - // TODO: not partial yet, currently skips refs 613 - 614 - const qresult = queryPostThread(jsonTyped.uri) 615 - if (!qresult) { 616 - return new Response( 617 - JSON.stringify({ 618 - error: "Feed not found", 619 - }), 620 - { 621 - status: 404, 622 - headers: withCors({ "Content-Type": "application/json" }), 623 - } 624 - ); 625 - } 626 - const response: IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.OutputSchema = 627 - qresult 628 - 629 - return new Response(JSON.stringify(response), { 630 - headers: withCors({ "Content-Type": "application/json" }), 631 - }); 632 - } 633 - case "party.whey.app.bsky.feed.getQuotesPartial": { 634 - const jsonTyped = 635 - jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetQuotesPartial.QueryParams; 636 - 637 - // TODO: not partial yet, currently skips refs 638 - 639 - const qresult = queryQuotes(jsonTyped.uri) 640 - if (!qresult) { 641 - return new Response( 642 - JSON.stringify({ 643 - error: "Feed not found", 644 - }), 645 - { 646 - status: 404, 647 - headers: withCors({ "Content-Type": "application/json" }), 648 - } 649 - ); 650 - } 651 - const response: IndexServerTypes.PartyWheyAppBskyFeedGetQuotesPartial.OutputSchema = 652 - { 653 - uri: jsonTyped.uri, 654 - posts: qresult.map((feedviewpost)=>{return feedviewpost.post as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView>}) 655 - }; 656 - 657 - return new Response(JSON.stringify(response), { 658 - headers: withCors({ "Content-Type": "application/json" }), 659 - }); 660 - } 661 - case "party.whey.app.bsky.feed.getRepostedByPartial": { 662 - const jsonTyped = 663 - jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetRepostedByPartial.QueryParams; 664 - 665 - // TODO: not partial yet, currently skips refs 666 - 667 - const qresult = queryReposts(jsonTyped.uri) 668 - if (!qresult) { 669 - return new Response( 670 - JSON.stringify({ 671 - error: "Feed not found", 672 - }), 673 - { 674 - status: 404, 675 - headers: withCors({ "Content-Type": "application/json" }), 676 - } 677 - ); 678 - } 679 - const response: IndexServerTypes.PartyWheyAppBskyFeedGetRepostedByPartial.OutputSchema = 680 - { 681 - uri: jsonTyped.uri, 682 - repostedBy: qresult as ATPAPI.$Typed<ATPAPI.AppBskyActorDefs.ProfileView>[] 683 - }; 684 - 685 - return new Response(JSON.stringify(response), { 686 - headers: withCors({ "Content-Type": "application/json" }), 687 - }); 688 - } 689 - // TODO: too hard for now 690 - // case "party.whey.app.bsky.feed.getListFeedPartial": { 691 - // const jsonTyped = 692 - // jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetListFeedPartial.QueryParams; 693 - 694 - // const response: IndexServerTypes.PartyWheyAppBskyFeedGetListFeedPartial.OutputSchema = 695 - // {}; 696 - 697 - // return new Response(JSON.stringify(response), { 698 - // headers: withCors({ "Content-Type": "application/json" }), 699 - // }); 700 - // } 701 - /* three more coming soon 702 - app.bsky.graph.getLists 703 - app.bsky.graph.getList 704 - app.bsky.graph.getActorStarterPacks 705 - */ 706 - default: { 707 - return new Response( 708 - JSON.stringify({ 709 - error: "XRPCNotSupported", 710 - message: 711 - "HEY hello there my name is whey dot party and you have used my custom appview that is very cool but have you considered that XRPC Not Supported", 712 - }), 713 - { 714 - status: 404, 715 - headers: withCors({ "Content-Type": "application/json" }), 716 - } 717 - ); 718 - } 719 - } 720 - 721 - // return new Response("Not Found", { status: 404 }); 722 - } 723 - 724 1773 type linksQuery = { 725 1774 target: string; 726 1775 collection: string; ··· 795 1844 return typeof str === "string" && str.startsWith("did:"); 796 1845 } 797 1846 798 - export async function constellationAPIHandler(req: Request): Promise<Response> { 799 - const url = new URL(req.url); 800 - const pathname = url.pathname; 801 - const searchParams = searchParamsToJson(url.searchParams) as linksQuery; 802 - const jsonUntyped = searchParams; 803 - 804 - if (!jsonUntyped.target) { 805 - return new Response( 806 - JSON.stringify({ error: "Missing required parameter: target" }), 807 - { 808 - status: 400, 809 - headers: withCors({ "Content-Type": "application/json" }), 810 - } 811 - ); 812 - } 813 - 814 - const did = isDid(searchParams.target) 815 - ? searchParams.target 816 - : new AtUri(searchParams.target).host; 817 - const db = getDbForDid(did); 818 - if (!db) { 819 - return new Response( 820 - JSON.stringify({ 821 - error: "User not found", 822 - }), 823 - { 824 - status: 404, 825 - headers: withCors({ "Content-Type": "application/json" }), 826 - } 827 - ); 828 - } 829 - 830 - const limit = 16; //Math.min(parseInt(searchParams.limit || "50", 10), 100); 831 - const offset = parseInt(searchParams.cursor || "0", 10); 832 - 833 - switch (pathname) { 834 - case "/links": { 835 - const jsonTyped = jsonUntyped as linksQuery; 836 - if (!jsonTyped.collection || !jsonTyped.path) { 837 - return new Response( 838 - JSON.stringify({ 839 - error: "Missing required parameters: collection, path", 840 - }), 841 - { 842 - status: 400, 843 - headers: withCors({ "Content-Type": "application/json" }), 844 - } 845 - ); 846 - } 847 - 848 - const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 849 - /^\./, 850 - "" 851 - )}`; 852 - 853 - const paginatedSql = `${SQL.links} LIMIT ? OFFSET ?`; 854 - const rows = db 855 - .prepare(paginatedSql) 856 - .all(jsonTyped.target, jsonTyped.collection, field, limit, offset); 857 - 858 - const countResult = db 859 - .prepare(SQL.count) 860 - .get(jsonTyped.target, jsonTyped.collection, field); 861 - const total = countResult ? Number(countResult.total) : 0; 862 - 863 - const linking_records: linksRecord[] = rows.map((row: any) => { 864 - const rkey = row.srcuri.split("/").pop()!; 865 - return { 866 - did: row.srcdid, 867 - collection: row.srccol, 868 - rkey, 869 - }; 870 - }); 871 - 872 - const response: linksRecordsResponse = { 873 - total: total.toString(), 874 - linking_records, 875 - }; 876 - 877 - const nextCursor = offset + linking_records.length; 878 - if (nextCursor < total) { 879 - response.cursor = nextCursor.toString(); 880 - } 881 - 882 - return new Response(JSON.stringify(response), { 883 - headers: withCors({ "Content-Type": "application/json" }), 884 - }); 885 - } 886 - case "/links/distinct-dids": { 887 - const jsonTyped = jsonUntyped as linksQuery; 888 - if (!jsonTyped.collection || !jsonTyped.path) { 889 - return new Response( 890 - JSON.stringify({ 891 - error: "Missing required parameters: collection, path", 892 - }), 893 - { 894 - status: 400, 895 - headers: withCors({ "Content-Type": "application/json" }), 896 - } 897 - ); 898 - } 899 - 900 - const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 901 - /^\./, 902 - "" 903 - )}`; 904 - 905 - const paginatedSql = `${SQL.distinctDids} LIMIT ? OFFSET ?`; 906 - const rows = db 907 - .prepare(paginatedSql) 908 - .all(jsonTyped.target, jsonTyped.collection, field, limit, offset); 909 - 910 - const countResult = db 911 - .prepare(SQL.countDistinctDids) 912 - .get(jsonTyped.target, jsonTyped.collection, field); 913 - const total = countResult ? Number(countResult.total) : 0; 914 - 915 - const linking_dids: string[] = rows.map((row: any) => row.srcdid); 916 - 917 - const response: linksDidsResponse = { 918 - total: total.toString(), 919 - linking_dids, 920 - }; 921 - 922 - const nextCursor = offset + linking_dids.length; 923 - if (nextCursor < total) { 924 - response.cursor = nextCursor.toString(); 925 - } 926 - 927 - return new Response(JSON.stringify(response), { 928 - headers: withCors({ "Content-Type": "application/json" }), 929 - }); 930 - } 931 - case "/links/count": { 932 - const jsonTyped = jsonUntyped as linksQuery; 933 - if (!jsonTyped.collection || !jsonTyped.path) { 934 - return new Response( 935 - JSON.stringify({ 936 - error: "Missing required parameters: collection, path", 937 - }), 938 - { 939 - status: 400, 940 - headers: withCors({ "Content-Type": "application/json" }), 941 - } 942 - ); 943 - } 944 - 945 - const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 946 - /^\./, 947 - "" 948 - )}`; 949 - 950 - const result = db 951 - .prepare(SQL.count) 952 - .get(jsonTyped.target, jsonTyped.collection, field); 953 - 954 - const response: linksCountResponse = { 955 - total: result && result.total ? result.total.toString() : "0", 956 - }; 957 - 958 - return new Response(JSON.stringify(response), { 959 - headers: withCors({ "Content-Type": "application/json" }), 960 - }); 961 - } 962 - case "/links/count/distinct-dids": { 963 - const jsonTyped = jsonUntyped as linksQuery; 964 - if (!jsonTyped.collection || !jsonTyped.path) { 965 - return new Response( 966 - JSON.stringify({ 967 - error: "Missing required parameters: collection, path", 968 - }), 969 - { 970 - status: 400, 971 - headers: withCors({ "Content-Type": "application/json" }), 972 - } 973 - ); 974 - } 975 - 976 - const field = `${jsonTyped.collection}:${jsonTyped.path.replace( 977 - /^\./, 978 - "" 979 - )}`; 980 - 981 - const result = db 982 - .prepare(SQL.countDistinctDids) 983 - .get(jsonTyped.target, jsonTyped.collection, field); 984 - 985 - const response: linksCountResponse = { 986 - total: result && result.total ? result.total.toString() : "0", 987 - }; 988 - 989 - return new Response(JSON.stringify(response), { 990 - headers: withCors({ "Content-Type": "application/json" }), 991 - }); 992 - } 993 - case "/links/all": { 994 - const jsonTyped = jsonUntyped as linksAllQuery; 995 - 996 - const rows = db.prepare(SQL.all).all(jsonTyped.target) as any[]; 997 - 998 - const links: linksAllResponse["links"] = {}; 999 - 1000 - for (const row of rows) { 1001 - if (!links[row.suburi]) { 1002 - links[row.suburi] = {}; 1003 - } 1004 - links[row.suburi][row.srccol] = { 1005 - records: row.records, 1006 - distinct_dids: row.distinct_dids, 1007 - }; 1008 - } 1009 - 1010 - const response: linksAllResponse = { 1011 - links, 1012 - }; 1013 - 1014 - return new Response(JSON.stringify(response), { 1015 - headers: withCors({ "Content-Type": "application/json" }), 1016 - }); 1017 - } 1018 - default: { 1019 - return new Response( 1020 - JSON.stringify({ 1021 - error: "NotSupported", 1022 - message: 1023 - "The requested endpoint is not supported by this Constellation implementation.", 1024 - }), 1025 - { 1026 - status: 404, 1027 - headers: withCors({ "Content-Type": "application/json" }), 1028 - } 1029 - ); 1030 - } 1031 - } 1032 - } 1033 - 1034 1847 function isImageEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedImages.Main { 1035 1848 return ( 1036 1849 typeof embed === "object" && ··· 1096 1909 if (isRecordWithMediaEmbed(embed)) return embed.record.record.uri; 1097 1910 return null; 1098 1911 } 1099 - 1100 - export function indexServerIndexer(ctx: indexHandlerContext) { 1101 - const record = assertRecord(ctx.value); 1102 - //const record = validateRecord(ctx.value); 1103 - const db = getDbForDid(ctx.doer); 1104 - if (!db) return; 1105 - console.log("indexering"); 1106 - switch (record?.$type) { 1107 - case "app.bsky.feed.like": { 1108 - return; 1109 - } 1110 - case "app.bsky.actor.profile": { 1111 - console.log("bsky profuile"); 1112 - 1113 - try { 1114 - const stmt = db.prepare(` 1115 - INSERT OR IGNORE INTO app_bsky_actor_profile ( 1116 - uri, did, cid, rev, createdat, indexedat, json, 1117 - displayname, 1118 - description, 1119 - avatarcid, 1120 - avatarmime, 1121 - bannercid, 1122 - bannermime 1123 - ) VALUES (?, ?, ?, ?, ?, ?, ?, 1124 - ?, ?, ?, 1125 - ?, ?, ?) 1126 - `); 1127 - console.log({ 1128 - uri: ctx.aturi, 1129 - did: ctx.doer, 1130 - cid: ctx.cid, 1131 - rev: ctx.rev, 1132 - createdat: record.createdAt, 1133 - indexedat: Date.now(), 1134 - json: JSON.stringify(record), 1135 - displayname: record.displayName, 1136 - description: record.description, 1137 - avatarcid: uncid(record.avatar?.ref), 1138 - avatarmime: record.avatar?.mimeType, 1139 - bannercid: uncid(record.banner?.ref), 1140 - bannermime: record.banner?.mimeType, 1141 - }); 1142 - stmt.run( 1143 - ctx.aturi ?? null, 1144 - ctx.doer ?? null, 1145 - ctx.cid ?? null, 1146 - ctx.rev ?? null, 1147 - record.createdAt ?? null, 1148 - Date.now(), 1149 - JSON.stringify(record), 1150 - 1151 - record.displayName ?? null, 1152 - record.description ?? null, 1153 - uncid(record.avatar?.ref) ?? null, 1154 - record.avatar?.mimeType ?? null, 1155 - uncid(record.banner?.ref) ?? null, 1156 - record.banner?.mimeType ?? null, 1157 - // TODO please add pinned posts 1158 - 1159 - ); 1160 - } catch (err) { 1161 - console.error("stmt.run failed:", err); 1162 - } 1163 - return; 1164 - } 1165 - case "app.bsky.feed.post": { 1166 - console.log("bsky post"); 1167 - const stmt = db.prepare(` 1168 - INSERT OR IGNORE INTO app_bsky_feed_post ( 1169 - uri, did, cid, rev, createdat, indexedat, json, 1170 - text, replyroot, replyparent, quote, 1171 - imagecount, image1cid, image1mime, image1aspect, 1172 - image2cid, image2mime, image2aspect, 1173 - image3cid, image3mime, image3aspect, 1174 - image4cid, image4mime, image4aspect, 1175 - videocount, videocid, videomime, videoaspect 1176 - ) VALUES (?, ?, ?, ?, ?, ?, ?, 1177 - ?, ?, ?, ?, 1178 - ?, ?, ?, ?, 1179 - ?, ?, ?, 1180 - ?, ?, ?, 1181 - ?, ?, ?, 1182 - ?, ?, ?, ?) 1183 - `); 1184 - 1185 - const embed = record.embed; 1186 - 1187 - const images = extractImages(embed); 1188 - const video = extractVideo(embed); 1189 - const quoteUri = extractQuoteUri(embed); 1190 - try { 1191 - stmt.run( 1192 - ctx.aturi ?? null, 1193 - ctx.doer ?? null, 1194 - ctx.cid ?? null, 1195 - ctx.rev ?? null, 1196 - record.createdAt, 1197 - Date.now(), 1198 - JSON.stringify(record), 1199 - 1200 - record.text ?? null, 1201 - record.reply?.root?.uri ?? null, 1202 - record.reply?.parent?.uri ?? null, 1203 - 1204 - quoteUri, 1205 - 1206 - images.length, 1207 - uncid(images[0]?.image?.ref) ?? null, 1208 - images[0]?.image?.mimeType ?? null, 1209 - images[0]?.aspectRatio && 1210 - images[0].aspectRatio.width && 1211 - images[0].aspectRatio.height 1212 - ? `${images[0].aspectRatio.width}:${images[0].aspectRatio.height}` 1213 - : null, 1214 - 1215 - uncid(images[1]?.image?.ref) ?? null, 1216 - images[1]?.image?.mimeType ?? null, 1217 - images[1]?.aspectRatio && 1218 - images[1].aspectRatio.width && 1219 - images[1].aspectRatio.height 1220 - ? `${images[1].aspectRatio.width}:${images[1].aspectRatio.height}` 1221 - : null, 1222 - 1223 - uncid(images[2]?.image?.ref) ?? null, 1224 - images[2]?.image?.mimeType ?? null, 1225 - images[2]?.aspectRatio && 1226 - images[2].aspectRatio.width && 1227 - images[2].aspectRatio.height 1228 - ? `${images[2].aspectRatio.width}:${images[2].aspectRatio.height}` 1229 - : null, 1230 - 1231 - uncid(images[3]?.image?.ref) ?? null, 1232 - images[3]?.image?.mimeType ?? null, 1233 - images[3]?.aspectRatio && 1234 - images[3].aspectRatio.width && 1235 - images[3].aspectRatio.height 1236 - ? `${images[3].aspectRatio.width}:${images[3].aspectRatio.height}` 1237 - : null, 1238 - 1239 - uncid(video?.video) ? 1 : 0, 1240 - uncid(video?.video) ?? null, 1241 - uncid(video?.video) ? "video/mp4" : null, 1242 - video?.aspectRatio 1243 - ? `${video.aspectRatio.width}:${video.aspectRatio.height}` 1244 - : null 1245 - ); 1246 - } catch (err) { 1247 - console.error("stmt.run failed:", err); 1248 - } 1249 - return; 1250 - } 1251 - default: { 1252 - // what the hell 1253 - return; 1254 - } 1255 - } 1256 - } 1257 - 1258 - // user data 1259 - function queryProfileView( 1260 - did: string, 1261 - type: "" 1262 - ): ATPAPI.AppBskyActorDefs.ProfileView | undefined; 1263 - function queryProfileView( 1264 - did: string, 1265 - type: "Basic" 1266 - ): ATPAPI.AppBskyActorDefs.ProfileViewBasic | undefined; 1267 - function queryProfileView( 1268 - did: string, 1269 - type: "Detailed" 1270 - ): ATPAPI.AppBskyActorDefs.ProfileViewDetailed | undefined; 1271 - function queryProfileView( 1272 - did: string, 1273 - type: "" | "Basic" | "Detailed" 1274 - ): 1275 - | ATPAPI.AppBskyActorDefs.ProfileView 1276 - | ATPAPI.AppBskyActorDefs.ProfileViewBasic 1277 - | ATPAPI.AppBskyActorDefs.ProfileViewDetailed 1278 - | undefined { 1279 - if (!isRegisteredIndexUser(did)) return; 1280 - const db = getDbForDid(did); 1281 - if (!db) return; 1282 - 1283 - const stmt = db.prepare(` 1284 - SELECT * 1285 - FROM app_bsky_actor_profile 1286 - WHERE did = ? 1287 - LIMIT 1; 1288 - `); 1289 - 1290 - const row = stmt.get(did) as ProfileRow; 1291 - 1292 - // simulate different types returned 1293 - switch (type) { 1294 - case "": { 1295 - const result: ATPAPI.AppBskyActorDefs.ProfileView = { 1296 - $type: "app.bsky.actor.defs#profileView", 1297 - did: did, 1298 - handle: "idiot.fuck.shit.example.com", // TODO: Resolve user identity here for the handle 1299 - displayName: row.displayname ?? undefined, 1300 - description: row.description ?? undefined, 1301 - avatar: "https://google.com/", // create profile URL from resolved identity 1302 - //associated?: ProfileAssociated, 1303 - indexedAt: row.createdat 1304 - ? new Date(row.createdat).toISOString() 1305 - : undefined, 1306 - createdAt: row.createdat 1307 - ? new Date(row.createdat).toISOString() 1308 - : undefined, 1309 - //viewer?: ViewerState, 1310 - //labels?: ComAtprotoLabelDefs.Label[], 1311 - //verification?: VerificationState, 1312 - //status?: StatusView, 1313 - }; 1314 - return result; 1315 - } 1316 - case "Basic": { 1317 - const result: ATPAPI.AppBskyActorDefs.ProfileViewBasic = { 1318 - $type: "app.bsky.actor.defs#profileViewBasic", 1319 - did: did, 1320 - handle: "idiot.fuck.shit.example.com", // TODO: Resolve user identity here for the handle 1321 - displayName: row.displayname ?? undefined, 1322 - avatar: "https://google.com/", // create profile URL from resolved identity 1323 - //associated?: ProfileAssociated, 1324 - createdAt: row.createdat 1325 - ? new Date(row.createdat).toISOString() 1326 - : undefined, 1327 - //viewer?: ViewerState, 1328 - //labels?: ComAtprotoLabelDefs.Label[], 1329 - //verification?: VerificationState, 1330 - //status?: StatusView, 1331 - }; 1332 - return result; 1333 - } 1334 - case "Detailed": { 1335 - // Query for follower count from the backlink_skeleton table 1336 - const followersStmt = db.prepare(` 1337 - SELECT COUNT(*) as count 1338 - FROM backlink_skeleton 1339 - WHERE subdid = ? AND srccol = 'app.bsky.graph.follow' 1340 - `); 1341 - const followersResult = followersStmt.get(did) as { count: number }; 1342 - const followersCount = followersResult?.count ?? 0; 1343 - 1344 - // Query for following count from the app_bsky_graph_follow table 1345 - const followingStmt = db.prepare(` 1346 - SELECT COUNT(*) as count 1347 - FROM app_bsky_graph_follow 1348 - WHERE did = ? 1349 - `); 1350 - const followingResult = followingStmt.get(did) as { count: number }; 1351 - const followsCount = followingResult?.count ?? 0; 1352 - 1353 - // Query for post count from the app_bsky_feed_post table 1354 - const postsStmt = db.prepare(` 1355 - SELECT COUNT(*) as count 1356 - FROM app_bsky_feed_post 1357 - WHERE did = ? 1358 - `); 1359 - const postsResult = postsStmt.get(did) as { count: number }; 1360 - const postsCount = postsResult?.count ?? 0; 1361 - 1362 - const result: ATPAPI.AppBskyActorDefs.ProfileViewDetailed = { 1363 - $type: "app.bsky.actor.defs#profileViewDetailed", 1364 - did: did, 1365 - handle: "idiot.fuck.shit.example.com", // TODO: Resolve user identity here for the handle 1366 - displayName: row.displayname ?? undefined, 1367 - description: row.description ?? undefined, 1368 - avatar: "https://google.com/", // TODO: create profile URL from resolved identity 1369 - banner: "https://youtube.com/", // same here 1370 - followersCount: followersCount, 1371 - followsCount: followsCount, 1372 - postsCount: postsCount, 1373 - //associated?: ProfileAssociated, 1374 - //joinedViaStarterPack?: // AppBskyGraphDefs.StarterPackViewBasic; 1375 - indexedAt: row.createdat 1376 - ? new Date(row.createdat).toISOString() 1377 - : undefined, 1378 - createdAt: row.createdat 1379 - ? new Date(row.createdat).toISOString() 1380 - : undefined, 1381 - //viewer?: ViewerState, 1382 - //labels?: ComAtprotoLabelDefs.Label[], 1383 - pinnedPost: undefined, //row.; // TODO: i forgot to put pinnedp posts in db schema oops 1384 - //verification?: VerificationState, 1385 - //status?: StatusView, 1386 - }; 1387 - return result; 1388 - } 1389 - default: 1390 - throw new Error("Invalid type"); 1391 - } 1392 - } 1393 - 1394 - // post hydration 1395 - function queryPostView( 1396 - uri: string 1397 - ): ATPAPI.AppBskyFeedDefs.PostView | undefined { 1398 - const URI = new AtUri(uri); 1399 - const did = URI.host; 1400 - if (!isRegisteredIndexUser(did)) return; 1401 - const db = getDbForDid(did); 1402 - if (!db) return; 1403 - 1404 - const stmt = db.prepare(` 1405 - SELECT * 1406 - FROM app_bsky_feed_post 1407 - WHERE uri = ? 1408 - LIMIT 1; 1409 - `); 1410 - 1411 - const row = stmt.get(uri) as PostRow; 1412 - const profileView = queryProfileView(did, "Basic"); 1413 - if (!row || !row.cid || !profileView || !row.json) return; 1414 - const value = JSON.parse(row.json) as ATPAPI.AppBskyFeedPost.Record; 1415 - 1416 - const post: ATPAPI.AppBskyFeedDefs.PostView = { 1417 - uri: row.uri, 1418 - cid: row.cid, 1419 - author: profileView, 1420 - record: value, 1421 - indexedAt: new Date(row.indexedat).toISOString(), 1422 - embed: value.embed, 1423 - }; 1424 - 1425 - return post; 1426 - } 1427 - function queryFeedViewPost( 1428 - uri: string 1429 - ): ATPAPI.AppBskyFeedDefs.FeedViewPost | undefined { 1430 - 1431 - const post = queryPostView(uri) 1432 - if (!post) return; 1433 - 1434 - const feedviewpost: ATPAPI.AppBskyFeedDefs.FeedViewPost = { 1435 - $type: 'app.bsky.feed.defs#feedViewPost', 1436 - post: post, 1437 - //reply: ReplyRef, 1438 - //reason: , 1439 - }; 1440 - 1441 - return feedviewpost; 1442 - } 1443 - 1444 - interface BaseRow { 1445 - uri: string; 1446 - did: string; 1447 - cid: string | null; 1448 - rev: string | null; 1449 - createdat: number | null; 1450 - indexedat: number; 1451 - json: string | null; 1452 - } 1453 - interface GeneratorRow extends BaseRow { 1454 - displayname: string | null; 1455 - description: string | null; 1456 - avatarcid: string | null; 1457 - } 1458 - interface LikeRow extends BaseRow { 1459 - subject: string; 1460 - } 1461 - interface RepostRow extends BaseRow { 1462 - subject: string; 1463 - } 1464 - interface BacklinkRow { 1465 - srcuri: string; 1466 - srcdid: string; 1467 - } 1468 - 1469 - const FEED_LIMIT = 50; 1470 - 1471 - // user feedgens 1472 - 1473 - function queryActorFeeds(did: string): ATPAPI.AppBskyFeedDefs.GeneratorView[] { 1474 - if (!isRegisteredIndexUser(did)) return []; 1475 - const db = getDbForDid(did); 1476 - if (!db) return []; 1477 - 1478 - const stmt = db.prepare(` 1479 - SELECT uri, cid, did, json, indexedat 1480 - FROM app_bsky_feed_generator 1481 - WHERE did = ? 1482 - ORDER BY createdat DESC; 1483 - `); 1484 - 1485 - const rows = stmt.all(did) as unknown as GeneratorRow[]; 1486 - const creatorView = queryProfileView(did, "Basic"); 1487 - if (!creatorView) return []; 1488 - 1489 - return rows 1490 - .map((row) => { 1491 - try { 1492 - if (!row.json) return; 1493 - const record = JSON.parse( 1494 - row.json 1495 - ) as ATPAPI.AppBskyFeedGenerator.Record; 1496 - return { 1497 - $type: "app.bsky.feed.defs#generatorView", 1498 - uri: row.uri, 1499 - cid: row.cid, 1500 - did: row.did, 1501 - creator: creatorView, 1502 - displayName: record.displayName, 1503 - description: record.description, 1504 - descriptionFacets: record.descriptionFacets, 1505 - avatar: record.avatar, 1506 - likeCount: 0, // TODO: this should be easy 1507 - indexedAt: new Date(row.indexedat).toISOString(), 1508 - } as ATPAPI.AppBskyFeedDefs.GeneratorView; 1509 - } catch { 1510 - return undefined; 1511 - } 1512 - }) 1513 - .filter((v): v is ATPAPI.AppBskyFeedDefs.GeneratorView => !!v); 1514 - } 1515 - 1516 - function queryFeedGenerator( 1517 - uri: string 1518 - ): ATPAPI.AppBskyFeedDefs.GeneratorView | undefined { 1519 - return queryFeedGenerators([uri])[0]; 1520 - } 1521 - 1522 - function queryFeedGenerators( 1523 - uris: string[] 1524 - ): ATPAPI.AppBskyFeedDefs.GeneratorView[] { 1525 - const generators: ATPAPI.AppBskyFeedDefs.GeneratorView[] = []; 1526 - const urisByDid = new Map<string, string[]>(); 1527 - 1528 - for (const uri of uris) { 1529 - try { 1530 - const { host: did } = new AtUri(uri); 1531 - if (!urisByDid.has(did)) { 1532 - urisByDid.set(did, []); 1533 - } 1534 - urisByDid.get(did)!.push(uri); 1535 - } catch { 1536 - } 1537 - } 1538 - 1539 - for (const [did, didUris] of urisByDid.entries()) { 1540 - if (!isRegisteredIndexUser(did)) continue; 1541 - const db = getDbForDid(did); 1542 - if (!db) continue; 1543 - 1544 - const placeholders = didUris.map(() => "?").join(","); 1545 - const stmt = db.prepare(` 1546 - SELECT uri, cid, did, json, indexedat 1547 - FROM app_bsky_feed_generator 1548 - WHERE uri IN (${placeholders}); 1549 - `); 1550 - 1551 - const rows = stmt.all(...didUris) as unknown as GeneratorRow[]; 1552 - if (rows.length === 0) continue; 1553 - 1554 - const creatorView = queryProfileView(did, ""); 1555 - if (!creatorView) continue; 1556 - 1557 - for (const row of rows) { 1558 - try { 1559 - if (!row.json || !row.cid ) continue; 1560 - const record = JSON.parse( 1561 - row.json 1562 - ) as ATPAPI.AppBskyFeedGenerator.Record; 1563 - generators.push({ 1564 - $type: "app.bsky.feed.defs#generatorView", 1565 - uri: row.uri, 1566 - cid: row.cid, 1567 - did: row.did, 1568 - creator: creatorView, 1569 - displayName: record.displayName, 1570 - description: record.description, 1571 - descriptionFacets: record.descriptionFacets, 1572 - avatar: record.avatar as string | undefined, 1573 - likeCount: 0, 1574 - indexedAt: new Date(row.indexedat).toISOString(), 1575 - }); 1576 - } catch {} 1577 - } 1578 - } 1579 - return generators; 1580 - } 1581 - 1582 - // user feeds 1583 - 1584 - function queryAuthorFeed( 1585 - did: string, 1586 - cursor?: string 1587 - ): 1588 - | { 1589 - items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1590 - cursor: string | undefined; 1591 - } 1592 - | undefined { 1593 - if (!isRegisteredIndexUser(did)) return; 1594 - const db = getDbForDid(did); 1595 - if (!db) return; 1596 - 1597 - // TODO: implement this for real 1598 - let query = ` 1599 - SELECT uri, indexedat, cid 1600 - FROM app_bsky_feed_post 1601 - WHERE did = ? 1602 - `; 1603 - const params: (string | number)[] = [did]; 1604 - 1605 - if (cursor) { 1606 - const [indexedat, cid] = cursor.split("::"); 1607 - query += ` AND (indexedat < ? OR (indexedat = ? AND cid < ?))`; 1608 - params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); 1609 - } 1610 - 1611 - query += ` ORDER BY indexedat DESC, cid DESC LIMIT ${FEED_LIMIT}`; 1612 - 1613 - const stmt = db.prepare(query); 1614 - const rows = stmt.all(...params) as { 1615 - uri: string; 1616 - indexedat: number; 1617 - cid: string; 1618 - }[]; 1619 - 1620 - const items = rows 1621 - .map((row) => queryFeedViewPost(row.uri)) // TODO: for replies and repost i should inject the reason here 1622 - .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1623 - 1624 - const lastItem = rows[rows.length - 1]; 1625 - const nextCursor = lastItem 1626 - ? `${lastItem.indexedat}::${lastItem.cid}` 1627 - : undefined; 1628 - 1629 - return { items, cursor: nextCursor }; 1630 - } 1631 - 1632 - function queryListFeed( 1633 - uri: string, 1634 - cursor?: string 1635 - ): 1636 - | { 1637 - items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1638 - cursor: string | undefined; 1639 - } 1640 - | undefined { 1641 - return { items: [], cursor: undefined }; 1642 - } 1643 - 1644 - function queryActorLikes( 1645 - did: string, 1646 - cursor?: string 1647 - ): 1648 - | { 1649 - items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1650 - cursor: string | undefined; 1651 - } 1652 - | undefined { 1653 - if (!isRegisteredIndexUser(did)) return; 1654 - const db = getDbForDid(did); 1655 - if (!db) return; 1656 - 1657 - let query = ` 1658 - SELECT subject, indexedat, cid 1659 - FROM app_bsky_feed_like 1660 - WHERE did = ? 1661 - `; 1662 - const params: (string | number)[] = [did]; 1663 - 1664 - if (cursor) { 1665 - const [indexedat, cid] = cursor.split("::"); 1666 - query += ` AND (indexedat < ? OR (indexedat = ? AND cid < ?))`; 1667 - params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); 1668 - } 1669 - 1670 - query += ` ORDER BY indexedat DESC, cid DESC LIMIT ${FEED_LIMIT}`; 1671 - 1672 - const stmt = db.prepare(query); 1673 - const rows = stmt.all(...params) as { 1674 - subject: string; 1675 - indexedat: number; 1676 - cid: string; 1677 - }[]; 1678 - 1679 - const items = rows 1680 - .map((row) => queryFeedViewPost(row.subject)) 1681 - .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1682 - 1683 - const lastItem = rows[rows.length - 1]; 1684 - const nextCursor = lastItem 1685 - ? `${lastItem.indexedat}::${lastItem.cid}` 1686 - : undefined; 1687 - 1688 - return { items, cursor: nextCursor }; 1689 - } 1690 - 1691 - // post metadata 1692 - 1693 - function queryLikes( 1694 - uri: string 1695 - ): ATPAPI.AppBskyFeedGetLikes.Like[] | undefined { 1696 - const postUri = new AtUri(uri); 1697 - const postAuthorDid = postUri.hostname; 1698 - if (!isRegisteredIndexUser(postAuthorDid)) return; 1699 - const db = getDbForDid(postAuthorDid); 1700 - if (!db) return; 1701 - 1702 - const stmt = db.prepare(` 1703 - SELECT b.srcdid, b.srcuri 1704 - FROM backlink_skeleton AS b 1705 - WHERE b.suburi = ? AND b.srccol = 'app_bsky_feed_like' 1706 - ORDER BY b.id DESC; 1707 - `); 1708 - 1709 - const rows = stmt.all(uri) as unknown as BacklinkRow[]; 1710 - 1711 - return rows 1712 - .map((row) => { 1713 - const actor = queryProfileView(row.srcdid, ""); 1714 - if (!actor) return; 1715 - 1716 - return { 1717 - // TODO write indexedAt for spacedust indexes 1718 - createdAt: new Date(Date.now()).toISOString(), 1719 - indexedAt: new Date(Date.now()).toISOString(), 1720 - actor: actor, 1721 - }; 1722 - }) 1723 - .filter((like): like is ATPAPI.AppBskyFeedGetLikes.Like => !!like); 1724 - } 1725 - 1726 - function queryReposts(uri: string): ATPAPI.AppBskyActorDefs.ProfileView[] { 1727 - const postUri = new AtUri(uri); 1728 - const postAuthorDid = postUri.hostname; 1729 - if (!isRegisteredIndexUser(postAuthorDid)) return []; 1730 - const db = getDbForDid(postAuthorDid); 1731 - if (!db) return []; 1732 - 1733 - const stmt = db.prepare(` 1734 - SELECT srcdid 1735 - FROM backlink_skeleton 1736 - WHERE suburi = ? AND srccol = 'app_bsky_feed_repost' 1737 - ORDER BY id DESC; 1738 - `); 1739 - 1740 - const rows = stmt.all(uri) as { srcdid: string }[]; 1741 - 1742 - return rows 1743 - .map((row) => queryProfileView(row.srcdid, "")) 1744 - .filter((p): p is ATPAPI.AppBskyActorDefs.ProfileView => !!p); 1745 - } 1746 - 1747 - function queryQuotes(uri: string): ATPAPI.AppBskyFeedDefs.FeedViewPost[] { 1748 - const postUri = new AtUri(uri); 1749 - const postAuthorDid = postUri.hostname; 1750 - if (!isRegisteredIndexUser(postAuthorDid)) return []; 1751 - const db = getDbForDid(postAuthorDid); 1752 - if (!db) return []; 1753 - 1754 - const stmt = db.prepare(` 1755 - SELECT srcuri 1756 - FROM backlink_skeleton 1757 - WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'quote' 1758 - ORDER BY id DESC; 1759 - `); 1760 - 1761 - const rows = stmt.all(uri) as { srcuri: string }[]; 1762 - 1763 - return rows 1764 - .map((row) => queryFeedViewPost(row.srcuri)) 1765 - .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1766 - } 1767 - 1768 - function queryPostThread( 1769 - uri: string 1770 - ): ATPAPI.AppBskyFeedGetPostThread.OutputSchema | undefined { 1771 - const post = queryPostView(uri); 1772 - if (!post) { 1773 - return { 1774 - thread: { 1775 - $type: "app.bsky.feed.defs#notFoundPost", 1776 - uri: uri, 1777 - notFound: true, 1778 - } as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.NotFoundPost> 1779 - } 1780 - } 1781 - 1782 - const thread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1783 - $type: "app.bsky.feed.defs#threadViewPost", 1784 - post: post, 1785 - replies: [], 1786 - }; 1787 - 1788 - let current = thread; 1789 - while ((current.post.record.reply as any)?.parent?.uri) { 1790 - const parentUri = (current.post.record.reply as any)?.parent?.uri; 1791 - const parentPost = queryPostView(parentUri); 1792 - if (!parentPost) break; 1793 - 1794 - const parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1795 - $type: "app.bsky.feed.defs#threadViewPost", 1796 - post: parentPost, 1797 - replies: [current as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>], 1798 - }; 1799 - current.parent = parentThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>; 1800 - current = parentThread; 1801 - } 1802 - 1803 - const seenUris = new Set<string>(); 1804 - const fetchReplies = ( 1805 - parentThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost 1806 - ) => { 1807 - if (seenUris.has(parentThread.post.uri)) return; 1808 - seenUris.add(parentThread.post.uri); 1809 - 1810 - const parentUri = new AtUri(parentThread.post.uri); 1811 - const parentAuthorDid = parentUri.hostname; 1812 - const db = getDbForDid(parentAuthorDid); 1813 - if (!db) return; 1814 - 1815 - const stmt = db.prepare(` 1816 - SELECT srcuri 1817 - FROM backlink_skeleton 1818 - WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'replyparent' 1819 - `); 1820 - const replyRows = stmt.all(parentThread.post.uri) as { srcuri: string }[]; 1821 - 1822 - const replies = replyRows 1823 - .map((row) => queryPostView(row.srcuri)) 1824 - .filter((p): p is ATPAPI.AppBskyFeedDefs.PostView => !!p); 1825 - 1826 - for (const replyPost of replies) { 1827 - const replyThread: ATPAPI.AppBskyFeedDefs.ThreadViewPost = { 1828 - $type: "app.bsky.feed.defs#threadViewPost", 1829 - post: replyPost, 1830 - parent: parentThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>, 1831 - replies: [], 1832 - }; 1833 - parentThread.replies?.push(replyThread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost>); 1834 - fetchReplies(replyThread); 1835 - } 1836 - }; 1837 - 1838 - fetchReplies(thread); 1839 - 1840 - const returned = thread as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.ThreadViewPost> 1841 - 1842 - return { thread: returned }; 1843 - }

+12 -8

main.ts

··· 14 14 import * as ATPAPI from "npm:@atproto/api"; 15 15 import { didDocument } from "./utils/diddoc.ts"; 16 16 import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts"; 17 - import { constellationAPIHandler, indexServerHandler, IndexServerUserManager } from "./indexserver.ts"; 17 + import { IndexServer, IndexServerConfig } from "./indexserver.ts" 18 18 import { viewServerHandler } from "./viewserver.ts"; 19 19 20 20 export const jetstreamurl = Deno.env.get("JETSTREAM_URL"); ··· 26 26 // AppView Setup 27 27 // ------------------------------------------ 28 28 29 - export const systemDB = new Database("system.db"); 30 - setupSystemDb(systemDB); 29 + const config: IndexServerConfig = { 30 + baseDbPath: './dbs', // The directory for user databases 31 + systemDbPath: './system.db', // The path for the main system database 32 + jetstreamUrl: jetstreamurl || "" 33 + }; 34 + const registeredUsersIndexServer = new IndexServer(config); 35 + setupSystemDb(registeredUsersIndexServer.systemDB); 31 36 32 37 // add me lol 33 - systemDB.exec(` 38 + registeredUsersIndexServer.systemDB.exec(` 34 39 INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) 35 40 VALUES ( 36 41 'did:plc:mn45tewwnse5btfftvd3powc', ··· 48 53 ); 49 54 `) 50 55 51 - export const indexerUserManager = new IndexServerUserManager(); 52 - indexerUserManager.coldStart(systemDB) 56 + registeredUsersIndexServer.start(); 53 57 54 58 // should do both of these per user actually, since now each user has their own db 55 59 // also the set of records and backlinks to listen should be seperate between index and view servers ··· 152 156 // return await viewServerHandler(req) 153 157 154 158 if (constellation) { 155 - return await constellationAPIHandler(req); 159 + return registeredUsersIndexServer.constellationAPIHandler(req); 156 160 } 157 161 158 162 if (indexServerRoutes.has(pathname)) { 159 - return await indexServerHandler(req); 163 + return registeredUsersIndexServer.indexServerHandler(req); 160 164 } else { 161 165 return await viewServerHandler(req); 162 166 }

+2 -2

readme.md

··· 1 1 # skylite (pre alpha) 2 - an attempt to make a lightweight, easily self-hostable, scoped appview 2 + an attempt to make a lightweight, easily self-hostable, scoped Bluesky appview 3 3 4 4 this project uses: 5 5 - live sync systems: [jetstream](https://github.com/bluesky-social/jetstream) and [spacedust](https://spacedust.microcosm.blue/) ··· 22 22 - its a backlink index so i only needed one table, and so it is complete 23 23 - Server: 24 24 - Initial implementation is done 25 - - uses per-user instantiaion thing so it can add or remove users as needed 25 + - uses per-user instantiation thing so it can add or remove users as needed 26 26 - pagination is not a thing yet \:\( 27 27 - does not implement the Ref / Partial routes yet (currently strips undefineds) (fixing this soon) 28 28 - also implements the entirety of the Constellation API routes as a bonus (under `/links/`)

+2 -1

utils/identity.ts

··· 1 1 2 2 import { DidResolver, HandleResolver } from "npm:@atproto/identity"; 3 - import { systemDB } from "../main.ts"; 3 + import { Database } from "jsr:@db/sqlite@0.11"; 4 + const systemDB = new Database("./system.db") // TODO: temporary shim. should seperate this to its own central system db instead of the now instantiated system dbs 4 5 type DidMethod = "web" | "plc"; 5 6 type DidDoc = { 6 7 "@context"?: unknown;