-20

.env

··· 1 1 - # main indexers 2 2 - JETSTREAM_URL="wss://jetstream1.us-east.bsky.network" 3 3 - SPACEDUST_URL="wss://spacedust.whey.party" 4 4 - 5 5 - # for backfill (useless if you just started the instance right now) 6 6 - CONSTELLATION_URL="https://constellation.microcosm.blue" 7 7 - # i dont actually know why i need this 8 8 - SLINGSHOT_URL="https://slingshot.whey.party" 9 9 - 10 10 - # bools 11 11 - INDEX_SERVER_ENABLED=true 12 12 - INDEX_SERVER_INVITES_REQUIRED=true 13 13 - 14 14 - VIEW_SERVER_ENABLED=true 15 15 - INDEX_SERVER_INVITES_REQUIRED=true 16 16 - 17 17 - # this is for both index and view server btw 18 18 - SERVICE_DID="did:web:local3768forumtest.whey.party" 19 19 - SERVICE_ENDPOINT="https://local3768forumtest.whey.party" 20 20 - SERVER_PORT="3768"

+3 -1

.gitignore

··· 4 4 docs/.vite/ 5 5 .gitignore 6 6 indexserver.ts 7 7 - dbs/ 7 7 + dbs/ 8 8 + config.jsonc 9 9 + config.json

+35

config.jsonc.example

··· 1 1 + { 2 2 + // Main indexers 3 3 + "jetstream": "wss://jetstream1.us-east.bsky.network", // you can self host it -> https://github.com/bluesky-social/jetstream 4 4 + "spacedust": "wss://spacedust.your.site", // you can self host it -> https://www.microcosm.blue 5 5 + 6 6 + // For backfill (optional) 7 7 + "constellation": "https://constellation.microcosm.blue", // (not useful on a new setup — requires pre-existing data to backfill) 8 8 + 9 9 + // Utility services 10 10 + "slingshot": "https://slingshot.your.site", // you can self host it -> https://www.microcosm.blue 11 11 + 12 12 + // Index Server config 13 13 + "indexServer": { 14 14 + "inviteOnly": true, 15 15 + "port": 3767, 16 16 + "did": "did:web:skyliteindexserver.your.site", // should be the same domain as the endpoint 17 17 + "host": "https://skyliteindexserver.your.site" 18 18 + }, 19 19 + 20 20 + // View Server config 21 21 + "viewServer": { 22 22 + "inviteOnly": true, 23 23 + "port": 3768, 24 24 + "did": "did:web:skyliteviewserver.your.site", // should be the same domain as the endpoint 25 25 + "host": "https://skyliteviewserver.your.site", 26 26 + 27 27 + // In order of which skylite index servers or bsky appviews to use first 28 28 + "indexPriority": [ 29 29 + "user#skylite_index", // user resolved skylite index server 30 30 + "did:web:backupindexserver.your.site#skylite_index", // a specific skylite index server 31 31 + "user#bsky_appview", // user resolved bsky appview 32 32 + "did:web:api.bsky.app#bsky_appview" // a specific bsky appview 33 33 + ] 34 34 + } 35 35 + }

+45

config.ts

··· 1 1 + import { parse } from "jsr:@std/jsonc"; 2 2 + import * as z from "npm:zod"; 3 3 + 4 4 + // configure these from the config.jsonc file (you can use config.jsonc.example as reference) 5 5 + const indexTarget = z.string().refine( 6 6 + (val) => { 7 7 + const parts = val.split("#"); 8 8 + if (parts.length !== 2) return false; 9 9 + 10 10 + const [prefix, suffix] = parts; 11 11 + const validPrefix = prefix === "user" || prefix.startsWith("did:web:"); 12 12 + const validSuffix = suffix === "skylite_index" || suffix === "bsky_appview"; 13 13 + 14 14 + return validPrefix && validSuffix; 15 15 + }, 16 16 + { 17 17 + message: 18 18 + "Each indexPriority entry must be in the form 'user#skylite_index', 'user#bsky_appview', 'did:web:...#skylite_index', or 'did:web:...#bsky_appview'", 19 19 + } 20 20 + ); 21 21 + 22 22 + const ConfigSchema = z.object({ 23 23 + jetstream: z.string(), 24 24 + spacedust: z.string(), 25 25 + constellation: z.string(), 26 26 + slingshot: z.string(), 27 27 + indexServer: z.object({ 28 28 + inviteOnly: z.boolean(), 29 29 + port: z.number(), 30 30 + did: z.string(), 31 31 + host: z.string(), 32 32 + }), 33 33 + viewServer: z.object({ 34 34 + inviteOnly: z.boolean(), 35 35 + port: z.number(), 36 36 + did: z.string(), 37 37 + host: z.string(), 38 38 + indexPriority: z.array(indexTarget), 39 39 + }), 40 40 + }); 41 41 + 42 42 + const raw = await Deno.readTextFile("config.jsonc"); 43 43 + const config = ConfigSchema.parse(parse(raw)); 44 44 + 45 45 + export { config };

+2 -1

deno.json

··· 1 1 { 2 2 "tasks": { 3 3 - "dev": "deno run --watch -A --env-file main.ts" 3 3 + "index": "deno run --watch -A --env-file main-index.ts", 4 4 + "view": "deno run --watch -A --env-file main-view.ts" 4 5 }, 5 6 "imports": { 6 7 "@std/assert": "jsr:@std/assert@1"

+2 -2

index/jetstream.ts

··· 1 1 import { Database } from "jsr:@db/sqlite@0.11"; 2 2 - import { handleIndex } from "../main.ts"; 2 2 + import { config } from "../config.ts"; 3 3 import { resolveRecordFromURI } from "../utils/records.ts"; 4 4 import { JetstreamManager } from "../utils/sharders.ts"; 5 5 ··· 29 29 }); 30 30 } 31 31 32 32 - export async function handleJetstream(msg: any) { 32 32 + export async function handleJetstream(msg: any, handleIndex: Function) { 33 33 console.log("Received Jetstream message: ", msg); 34 34 35 35 const op = msg.commit.operation;

+6 -4

index/onboardingBackfill.ts

··· 1 1 - import { indexServerIndexer } from "../indexserver.ts"; 2 2 - import { systemDB } from "../main.ts" 1 1 + import { genericIndexServer } from "../main-index.ts"; 2 2 + import { config } from "../config.ts" 3 3 import { FINEPDSAndHandleFromDid } from "../utils/identity.ts"; 4 4 5 5 ··· 68 68 const doer = did; 69 69 const rev = undefined; 70 70 const aturi = uri; 71 71 + const db = genericIndexServer.userManager.getDbForDid(doer); 72 72 + if (!db) return; 71 73 72 72 - indexServerIndexer({ 74 74 + genericIndexServer.indexServerIndexer({ 73 75 op, 74 76 doer, 75 77 rev, 76 78 aturi, 77 79 value, 78 80 indexsrc: "onboarding_backfill", 79 79 - userdbname: did, 81 81 + db: db, 80 82 }) 81 83 return; 82 84 // console.log(`[BACKFILL] ${collection} -> ${uri}`);

+1 -1

index/spacedust.ts

··· 1 1 import { Database } from "jsr:@db/sqlite@0.11"; 2 2 - import { handleIndex } from "../main.ts"; 2 2 + import { config } from "../config.ts"; 3 3 import { parseAtUri } from "../utils/aturi.ts"; 4 4 import { resolveRecordFromURI } from "../utils/records.ts"; 5 5 import { SpacedustManager } from "../utils/sharders.ts";

+56 -10

indexserver.ts

··· 5 5 import * as IndexServerTypes from "./utils/indexservertypes.ts"; 6 6 import { Database } from "jsr:@db/sqlite@0.11"; 7 7 import { setupUserDb } from "./utils/dbuser.ts"; 8 8 - // import { systemDB } from "./main.ts"; 8 8 + // import { systemDB } from "./env.ts"; 9 9 import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts"; 10 10 import { handleSpacedust, SpacedustLinkMessage } from "./index/spacedust.ts"; 11 11 import { handleJetstream } from "./index/jetstream.ts"; 12 12 import * as ATPAPI from "npm:@atproto/api"; 13 13 import { AtUri } from "npm:@atproto/api"; 14 14 import * as IndexServerAPI from "./indexclient/index.ts"; 15 15 + import * as IndexServerUtils from "./indexclient/util.ts" 15 16 16 17 export interface IndexServerConfig { 17 18 baseDbPath: string; ··· 77 78 return this.constellationAPIHandler(req); 78 79 } 79 80 return new Response("Not Found", { status: 404 }); 81 81 + } 82 82 + 83 83 + public handlesDid(did: string): boolean { 84 84 + return this.userManager.handlesDid(did); 80 85 } 81 86 82 87 // We will move all the global functions into this class as methods... ··· 268 273 269 274 // TODO: not partial yet, currently skips refs 270 275 271 271 - const qresult = this.queryActorLikes(jsonTyped.actor, jsonTyped.cursor); 276 276 + const qresult = this.queryActorLikesPartial(jsonTyped.actor, jsonTyped.cursor); 272 277 if (!qresult) { 273 278 return new Response( 274 279 JSON.stringify({ ··· 1027 1032 1028 1033 return post; 1029 1034 } 1035 1035 + 1036 1036 + constructPostViewRef(uri: string): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef { 1037 1037 + const post: IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef = { 1038 1038 + uri: uri, 1039 1039 + cid: "cid.invalid", // oh shit we dont know the cid TODO: major design flaw 1040 1040 + }; 1041 1041 + 1042 1042 + return post; 1043 1043 + } 1044 1044 + 1030 1045 queryFeedViewPost( 1031 1046 uri: string 1032 1047 ): ATPAPI.AppBskyFeedDefs.FeedViewPost | undefined { ··· 1041 1056 }; 1042 1057 1043 1058 return feedviewpost; 1059 1059 + } 1060 1060 + 1061 1061 + constructFeedViewPostRef( 1062 1062 + uri: string 1063 1063 + ): IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef { 1064 1064 + const post = this.constructPostViewRef(uri); 1065 1065 + 1066 1066 + const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = { 1067 1067 + $type: "party.whey.app.bsky.feed.defs#feedViewPostRef", 1068 1068 + post: post as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, 1069 1069 + } 1070 1070 + 1071 1071 + return feedviewpostref 1044 1072 } 1045 1073 1046 1074 // user feedgens ··· 1213 1241 return { items: [], cursor: undefined }; 1214 1242 } 1215 1243 1216 1216 - queryActorLikes( 1244 1244 + queryActorLikesPartial( 1217 1245 did: string, 1218 1246 cursor?: string 1219 1247 ): 1220 1248 | { 1221 1221 - items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; 1249 1249 + items: (ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef)[]; 1222 1250 cursor: string | undefined; 1223 1251 } 1224 1252 | undefined { 1253 1253 + // early return only if the actor did is not registered 1225 1254 if (!this.isRegisteredIndexUser(did)) return; 1226 1255 const db = this.userManager.getDbForDid(did); 1227 1256 if (!db) return; ··· 1249 1278 }[]; 1250 1279 1251 1280 const items = rows 1252 1252 - .map((row) => this.queryFeedViewPost(row.subject)) 1253 1253 - .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); 1281 1281 + .map((row) => { 1282 1282 + const subjectDid = new AtUri(row.subject).host; 1283 1283 + 1284 1284 + if (this.handlesDid(subjectDid)) { 1285 1285 + return this.queryFeedViewPost(row.subject); 1286 1286 + } else { 1287 1287 + return this.constructFeedViewPostRef(row.subject); 1288 1288 + } 1289 1289 + }) 1290 1290 + .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef => !!p); 1254 1291 1255 1292 const lastItem = rows[rows.length - 1]; 1256 1293 const nextCursor = lastItem ··· 1418 1455 1419 1456 return { thread: returned }; 1420 1457 } 1421 1421 - 1458 1458 + 1422 1459 /** 1423 1460 * please do not use this, use openDbForDid() instead 1424 1461 * @param did ··· 1431 1468 //await db.exec(/* CREATE IF NOT EXISTS statements */); 1432 1469 return db; 1433 1470 } 1434 1434 - 1471 1471 + /** 1472 1472 + * @deprecated use handlesDid() instead 1473 1473 + * @param did 1474 1474 + * @returns 1475 1475 + */ 1435 1476 isRegisteredIndexUser(did: string): boolean { 1436 1477 const stmt = this.systemDB.prepare(` 1437 1478 SELECT 1 ··· 1453 1494 this.indexServer = indexServer; 1454 1495 } 1455 1496 1456 1456 - private users = new Map<string, UserIndexServer>(); 1497 1497 + public users = new Map<string, UserIndexServer>(); 1498 1498 + public handlesDid(did: string): boolean { 1499 1499 + return this.users.has(did); 1500 1500 + } 1457 1501 1458 1502 /*async*/ addUser(did: string) { 1459 1503 if (this.users.has(did)) return; ··· 1508 1552 constructor(indexServerUserManager: IndexServerUserManager, did: string) { 1509 1553 this.did = did; 1510 1554 this.indexServerUserManager = indexServerUserManager; 1511 1511 - this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid(this.did); 1555 1555 + this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid( 1556 1556 + this.did 1557 1557 + ); 1512 1558 // should probably put the params of exactly what were listening to here 1513 1559 this.jetstream = new JetstreamManager((msg) => { 1514 1560 console.log("Received Jetstream message: ", msg);

+125

main-index.ts

··· 1 1 + import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts"; 2 2 + import { setupSystemDb } from "./utils/dbsystem.ts"; 3 3 + import { didDocument } from "./utils/diddoc.ts"; 4 4 + import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts"; 5 5 + import { IndexServer, IndexServerConfig } from "./indexserver.ts" 6 6 + import { extractDid } from "./utils/identity.ts"; 7 7 + import { config } from "./config.ts"; 8 8 + 9 9 + // ------------------------------------------ 10 10 + // AppView Setup 11 11 + // ------------------------------------------ 12 12 + 13 13 + const indexServerConfig: IndexServerConfig = { 14 14 + baseDbPath: './dbs/registered-users', // The directory for user databases 15 15 + systemDbPath: './dbs/registered-users/system.db', // The path for the main system database 16 16 + jetstreamUrl: config.jetstream 17 17 + }; 18 18 + export const genericIndexServer = new IndexServer(indexServerConfig); 19 19 + setupSystemDb(genericIndexServer.systemDB); 20 20 + 21 21 + // add me lol 22 22 + genericIndexServer.systemDB.exec(` 23 23 + INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) 24 24 + VALUES ( 25 25 + 'did:plc:mn45tewwnse5btfftvd3powc', 26 26 + 'admin', 27 27 + datetime('now'), 28 28 + 'ready' 29 29 + ); 30 30 + 31 31 + INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) 32 32 + VALUES ( 33 33 + 'did:web:did12.whey.party', 34 34 + 'admin', 35 35 + datetime('now'), 36 36 + 'ready' 37 37 + ); 38 38 + `) 39 39 + 40 40 + genericIndexServer.start(); 41 41 + 42 42 + // ------------------------------------------ 43 43 + // XRPC Method Implementations 44 44 + // ------------------------------------------ 45 45 + 46 46 + // const indexServerRoutes = new Set([ 47 47 + // "/xrpc/app.bsky.actor.getProfile", 48 48 + // "/xrpc/app.bsky.actor.getProfiles", 49 49 + // "/xrpc/app.bsky.feed.getActorFeeds", 50 50 + // "/xrpc/app.bsky.feed.getFeedGenerator", 51 51 + // "/xrpc/app.bsky.feed.getFeedGenerators", 52 52 + // "/xrpc/app.bsky.feed.getPosts", 53 53 + // "/xrpc/party.whey.app.bsky.feed.getActorLikesPartial", 54 54 + // "/xrpc/party.whey.app.bsky.feed.getAuthorFeedPartial", 55 55 + // "/xrpc/party.whey.app.bsky.feed.getLikesPartial", 56 56 + // "/xrpc/party.whey.app.bsky.feed.getPostThreadPartial", 57 57 + // "/xrpc/party.whey.app.bsky.feed.getQuotesPartial", 58 58 + // "/xrpc/party.whey.app.bsky.feed.getRepostedByPartial", 59 59 + // // more federated endpoints, not planned yet, lexicons will come later 60 60 + // /* 61 61 + // app.bsky.graph.getLists // doesnt need to because theres no items[], and its self ProfileViewBasic 62 62 + // app.bsky.graph.getList // needs to be Partial-ed (items[] union with ProfileViewRef) 63 63 + // app.bsky.graph.getActorStarterPacks // maybe doesnt need to be Partial-ed because its self ProfileViewBasic 64 64 + 65 65 + // app.bsky.feed.getListFeed // uhh actually already exists its getListFeedPartial 66 66 + // */ 67 67 + // "/xrpc/party.whey.app.bsky.feed.getListFeedPartial", 68 68 + // ]); 69 69 + 70 70 + Deno.serve( 71 71 + { port: config.indexServer.port }, 72 72 + (req: Request): Response => { 73 73 + const url = new URL(req.url); 74 74 + const pathname = url.pathname; 75 75 + const searchParams = searchParamsToJson(url.searchParams); 76 76 + 77 77 + if (pathname === "/.well-known/did.json") { 78 78 + return new Response(JSON.stringify(didDocument("index",config.indexServer.did,config.indexServer.host,"whatever")), { 79 79 + headers: withCors({ "Content-Type": "application/json" }), 80 80 + }); 81 81 + } 82 82 + if (pathname === "/health") { 83 83 + return new Response("OK", { 84 84 + status: 200, 85 85 + headers: withCors({ 86 86 + "Content-Type": "text/plain", 87 87 + }), 88 88 + }); 89 89 + } 90 90 + if (req.method === "OPTIONS") { 91 91 + return new Response(null, { 92 92 + status: 204, 93 93 + headers: { 94 94 + "Access-Control-Allow-Origin": "*", 95 95 + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", 96 96 + "Access-Control-Allow-Headers": "*", 97 97 + }, 98 98 + }); 99 99 + } 100 100 + console.log(`request for "${pathname}"`) 101 101 + const constellation = pathname.startsWith("/links") 102 102 + 103 103 + if (constellation) { 104 104 + const target = searchParams?.target as string 105 105 + const safeDid = extractDid(target); 106 106 + const targetserver = genericIndexServer.handlesDid(safeDid) 107 107 + if (targetserver) { 108 108 + return genericIndexServer.constellationAPIHandler(req); 109 109 + } else { 110 110 + return new Response( 111 111 + JSON.stringify({ 112 112 + error: "User not found", 113 113 + }), 114 114 + { 115 115 + status: 404, 116 116 + headers: withCors({ "Content-Type": "application/json" }), 117 117 + } 118 118 + ); 119 119 + } 120 120 + } else { 121 121 + // indexServerRoutes.has(pathname) 122 122 + return genericIndexServer.indexServerHandler(req); 123 123 + } 124 124 + } 125 125 + );

+59

main-view.ts

··· 1 1 + import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts"; 2 2 + import { setupSystemDb } from "./utils/dbsystem.ts"; 3 3 + import { didDocument } from "./utils/diddoc.ts"; 4 4 + import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts"; 5 5 + import { IndexServer, IndexServerConfig } from "./indexserver.ts" 6 6 + import { extractDid } from "./utils/identity.ts"; 7 7 + import { config } from "./config.ts"; 8 8 + import { viewServerHandler } from "./viewserver.ts"; 9 9 + 10 10 + // ------------------------------------------ 11 11 + // AppView Setup 12 12 + // ------------------------------------------ 13 13 + 14 14 + setupAuth({ 15 15 + serviceDid: config.viewServer.did, 16 16 + //keyCacheSize: 500, 17 17 + //keyCacheTTL: 10 * 60 * 1000, 18 18 + }); 19 19 + 20 20 + // ------------------------------------------ 21 21 + // XRPC Method Implementations 22 22 + // ------------------------------------------ 23 23 + 24 24 + 25 25 + Deno.serve( 26 26 + { port: config.viewServer.port }, 27 27 + async (req: Request): Promise<Response> => { 28 28 + const url = new URL(req.url); 29 29 + const pathname = url.pathname; 30 30 + const searchParams = searchParamsToJson(url.searchParams); 31 31 + 32 32 + if (pathname === "/.well-known/did.json") { 33 33 + return new Response(JSON.stringify(didDocument), { 34 34 + headers: withCors({ "Content-Type": "application/json" }), 35 35 + }); 36 36 + } 37 37 + if (pathname === "/health") { 38 38 + return new Response("OK", { 39 39 + status: 200, 40 40 + headers: withCors({ 41 41 + "Content-Type": "text/plain", 42 42 + }), 43 43 + }); 44 44 + } 45 45 + if (req.method === "OPTIONS") { 46 46 + return new Response(null, { 47 47 + status: 204, 48 48 + headers: { 49 49 + "Access-Control-Allow-Origin": "*", 50 50 + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", 51 51 + "Access-Control-Allow-Headers": "*", 52 52 + }, 53 53 + }); 54 54 + } 55 55 + console.log(`request for "${pathname}"`) 56 56 + 57 57 + return await viewServerHandler(req) 58 58 + } 59 59 + );

-172

main.ts

··· 1 1 - import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts"; 2 2 - import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts"; 3 3 - import { resolveRecordFromURI, validateRecord } from "./utils/records.ts"; 4 4 - import { setupSystemDb } from "./utils/dbsystem.ts"; 5 5 - import { setupUserDb } from "./utils/dbuser.ts"; 6 6 - import { handleSpacedust, startSpacedust } from "./index/spacedust.ts"; 7 7 - import { handleJetstream, startJetstream } from "./index/jetstream.ts"; 8 8 - import { Database } from "jsr:@db/sqlite@0.11"; 9 9 - //import express from "npm:express"; 10 10 - //import { createServer } from "./xrpc/index.ts"; 11 11 - import { indexHandlerContext } from "./index/types.ts"; 12 12 - import * as IndexServerTypes from "./utils/indexservertypes.ts"; 13 13 - import * as ViewServerTypes from "./utils/viewservertypes.ts"; 14 14 - import * as ATPAPI from "npm:@atproto/api"; 15 15 - import { didDocument } from "./utils/diddoc.ts"; 16 16 - import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts"; 17 17 - import { IndexServer, IndexServerConfig } from "./indexserver.ts" 18 18 - import { viewServerHandler } from "./viewserver.ts"; 19 19 - 20 20 - export const jetstreamurl = Deno.env.get("JETSTREAM_URL"); 21 21 - export const slingshoturl = Deno.env.get("SLINGSHOT_URL"); 22 22 - export const constellationurl = Deno.env.get("CONSTELLATION_URL"); 23 23 - export const spacedusturl = Deno.env.get("SPACEDUST_URL"); 24 24 - 25 25 - // ------------------------------------------ 26 26 - // AppView Setup 27 27 - // ------------------------------------------ 28 28 - 29 29 - const config: IndexServerConfig = { 30 30 - baseDbPath: './dbs', // The directory for user databases 31 31 - systemDbPath: './system.db', // The path for the main system database 32 32 - jetstreamUrl: jetstreamurl || "" 33 33 - }; 34 34 - const registeredUsersIndexServer = new IndexServer(config); 35 35 - setupSystemDb(registeredUsersIndexServer.systemDB); 36 36 - 37 37 - // add me lol 38 38 - registeredUsersIndexServer.systemDB.exec(` 39 39 - INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) 40 40 - VALUES ( 41 41 - 'did:plc:mn45tewwnse5btfftvd3powc', 42 42 - 'admin', 43 43 - datetime('now'), 44 44 - 'ready' 45 45 - ); 46 46 - 47 47 - INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) 48 48 - VALUES ( 49 49 - 'did:web:did12.whey.party', 50 50 - 'admin', 51 51 - datetime('now'), 52 52 - 'ready' 53 53 - ); 54 54 - `) 55 55 - 56 56 - registeredUsersIndexServer.start(); 57 57 - 58 58 - // should do both of these per user actually, since now each user has their own db 59 59 - // also the set of records and backlinks to listen should be seperate between index and view servers 60 60 - // damn 61 61 - // export const spacedustManager = new SpacedustManager((msg) => 62 62 - // handleSpacedust(msg) 63 63 - // ); 64 64 - // export const jetstreamManager = new JetstreamManager((msg) => 65 65 - // handleJetstream(msg) 66 66 - // ); 67 67 - // startSpacedust(); 68 68 - // startJetstream(); 69 69 - 70 70 - // 1. connect to system db 71 71 - // 2. get all registered users 72 72 - // parse config (maybe some are only indexes and maybe some are only views) 73 73 - // map all new jetstream and spacedust listeners 74 74 - // call handleIndex with the specific db to use 75 75 - 76 76 - setupAuth({ 77 77 - // local3768forumtest is just my tunnel from my dev env to the outside web that im reusing from forumtest 78 78 - serviceDid: `${Deno.env.get("SERVICE_DID")}`, 79 79 - //keyCacheSize: 500, 80 80 - //keyCacheTTL: 10 * 60 * 1000, 81 81 - }); 82 82 - 83 83 - // ------------------------------------------ 84 84 - // XRPC Method Implementations 85 85 - // ------------------------------------------ 86 86 - 87 87 - const indexServerRoutes = new Set([ 88 88 - "/xrpc/app.bsky.actor.getProfile", 89 89 - "/xrpc/app.bsky.actor.getProfiles", 90 90 - "/xrpc/app.bsky.feed.getActorFeeds", 91 91 - "/xrpc/app.bsky.feed.getFeedGenerator", 92 92 - "/xrpc/app.bsky.feed.getFeedGenerators", 93 93 - "/xrpc/app.bsky.feed.getPosts", 94 94 - "/xrpc/party.whey.app.bsky.feed.getActorLikesPartial", 95 95 - "/xrpc/party.whey.app.bsky.feed.getAuthorFeedPartial", 96 96 - "/xrpc/party.whey.app.bsky.feed.getLikesPartial", 97 97 - "/xrpc/party.whey.app.bsky.feed.getPostThreadPartial", 98 98 - "/xrpc/party.whey.app.bsky.feed.getQuotesPartial", 99 99 - "/xrpc/party.whey.app.bsky.feed.getRepostedByPartial", 100 100 - // more federated endpoints, not planned yet, lexicons will come later 101 101 - /* 102 102 - app.bsky.graph.getLists // doesnt need to because theres no items[], and its self ProfileViewBasic 103 103 - app.bsky.graph.getList // needs to be Partial-ed (items[] union with ProfileViewRef) 104 104 - app.bsky.graph.getActorStarterPacks // maybe doesnt need to be Partial-ed because its self ProfileViewBasic 105 105 - 106 106 - app.bsky.feed.getListFeed // uhh actually already exists its getListFeedPartial 107 107 - */ 108 108 - "/xrpc/party.whey.app.bsky.feed.getListFeedPartial", 109 109 - ]); 110 110 - 111 111 - Deno.serve( 112 112 - { port: Number(`${Deno.env.get("SERVER_PORT")}`) }, 113 113 - async (req: Request): Promise<Response> => { 114 114 - const url = new URL(req.url); 115 115 - const pathname = url.pathname; 116 116 - // const searchParams = searchParamsToJson(url.searchParams); 117 117 - // let reqBody: undefined | string; 118 118 - // let jsonbody: undefined | Record<string, unknown>; 119 119 - // try { 120 120 - // const clone = req.clone(); 121 121 - // jsonbody = await clone.json(); 122 122 - // } catch (e) { 123 123 - // console.warn("Request body is not valid JSON:", e); 124 124 - // } 125 125 - if (pathname === "/.well-known/did.json") { 126 126 - return new Response(JSON.stringify(didDocument), { 127 127 - headers: withCors({ "Content-Type": "application/json" }), 128 128 - }); 129 129 - } 130 130 - if (pathname === "/health") { 131 131 - return new Response("OK", { 132 132 - status: 200, 133 133 - headers: withCors({ 134 134 - "Content-Type": "text/plain", 135 135 - }), 136 136 - }); 137 137 - } 138 138 - if (req.method === "OPTIONS") { 139 139 - return new Response(null, { 140 140 - status: 204, 141 141 - headers: { 142 142 - "Access-Control-Allow-Origin": "*", 143 143 - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", 144 144 - "Access-Control-Allow-Headers": "*", 145 145 - }, 146 146 - }); 147 147 - } 148 148 - // const bskyUrl = `https://api.bsky.app${pathname}${url.search}`; 149 149 - // const hasAuth = req.headers.has("authorization"); 150 150 - // const xrpcMethod = pathname.startsWith("/xrpc/") 151 151 - // ? pathname.slice("/xrpc/".length) 152 152 - // : null; 153 153 - console.log(`request for "${pathname}"`) 154 154 - const constellation = pathname.startsWith("/links") 155 155 - 156 156 - // return await viewServerHandler(req) 157 157 - 158 158 - if (constellation) { 159 159 - return registeredUsersIndexServer.constellationAPIHandler(req); 160 160 - } 161 161 - 162 162 - if (indexServerRoutes.has(pathname)) { 163 163 - return registeredUsersIndexServer.indexServerHandler(req); 164 164 - } else { 165 165 - return await viewServerHandler(req); 166 166 - } 167 167 - } 168 168 - ); 169 169 - 170 170 - // ------------------------------------------ 171 171 - // Indexer 172 172 - // ------------------------------------------

+2 -2

readme.md

··· 44 44 this project is pre-alpha and not intended for general use yet. you are welcome to experiment if you dont mind errors or breaking changes. 45 45 46 46 the project is split into two, the "Index Server" and the "View Server". 47 47 - these currently run in a single process and share the same HTTP server and port. 47 47 + despite both living in this repo, they run different http servers with different configs 48 48 49 49 - configuration is in the `.env` file 49 49 + example configuration is in the `config.jsonc.example` file 50 50 51 51 expose your localhost to the web using a tunnel or something and use that url as the custom appview url 52 52

+37 -22

utils/diddoc.ts

··· 1 1 - export const didDocument = { 2 2 - "@context": [ 3 3 - "https://www.w3.org/ns/did/v1", 4 4 - "https://w3id.org/security/multikey/v1", 5 5 - ], 6 6 - id: `${Deno.env.get("SERVICE_DID")}`, 7 7 - verificationMethod: [ 8 8 - { 9 9 - id: `${Deno.env.get("SERVICE_DID")}#atproto`, 10 10 - type: "Multikey", 11 11 - controller: `${Deno.env.get("SERVICE_DID")}`, 12 12 - publicKeyMultibase: "bullshit", 1 1 + // type "both" should not be used 2 2 + export function didDocument( 3 3 + type: "view" | "index" | "both", 4 4 + did: string, 5 5 + endpoint: string, 6 6 + publicKeyMultibase: string, 7 7 + ) { 8 8 + const services = [ 9 9 + (type === "view" || type === "both") && { 10 10 + id: "#bsky_appview", 11 11 + type: "BskyAppView", 12 12 + serviceEndpoint: endpoint, 13 13 }, 14 14 - ], 15 15 - service: [ 16 16 - { 14 14 + (type === "view" || type === "both") && { 17 15 id: "#bsky_notif", 18 16 type: "BskyNotificationService", 19 19 - serviceEndpoint: `${Deno.env.get("SERVICE_ENDPOINT")}`, 17 17 + serviceEndpoint: endpoint, 20 18 }, 21 21 - { 22 22 - id: "#bsky_appview", 23 23 - type: "BskyAppView", 24 24 - serviceEndpoint: `${Deno.env.get("SERVICE_ENDPOINT")}`, 19 19 + (type === "index" || type === "both") && { 20 20 + id: "#skylite_index", 21 21 + type: "SkyliteIndexServer", 22 22 + serviceEndpoint: endpoint, 25 23 }, 26 26 - ], 27 27 - }; 24 24 + ].filter(Boolean); 25 25 + 26 26 + return { 27 27 + "@context": [ 28 28 + "https://www.w3.org/ns/did/v1", 29 29 + "https://w3id.org/security/multikey/v1", 30 30 + ], 31 31 + id: did, 32 32 + verificationMethod: [ 33 33 + { 34 34 + id: `${did}#atproto`, 35 35 + type: "Multikey", 36 36 + controller: did, 37 37 + publicKeyMultibase: publicKeyMultibase, 38 38 + }, 39 39 + ], 40 40 + service: services, 41 41 + }; 42 42 + }

+14

utils/identity.ts

··· 1 1 2 2 import { DidResolver, HandleResolver } from "npm:@atproto/identity"; 3 3 import { Database } from "jsr:@db/sqlite@0.11"; 4 4 + import { AtUri } from "npm:@atproto/api"; 4 5 const systemDB = new Database("./system.db") // TODO: temporary shim. should seperate this to its own central system db instead of the now instantiated system dbs 5 6 type DidMethod = "web" | "plc"; 6 7 type DidDoc = { ··· 224 225 } catch (err) { 225 226 console.error(`Failed to extract/store PDS and handle for '${did}':`, err); 226 227 return null; 228 228 + } 229 229 + } 230 230 + 231 231 + export function extractDid(input: string): string { 232 232 + if (input.startsWith('did:')) { 233 233 + return input 234 234 + } 235 235 + 236 236 + try { 237 237 + const uri = new AtUri(input) 238 238 + return uri.host 239 239 + } catch (e) { 240 240 + throw new Error(`Invalid input: expected a DID or a valid AT URI, got "${input}"`) 227 241 } 228 242 }

+4 -4

utils/server.ts

··· 1 1 import ky from "npm:ky"; 2 2 import QuickLRU from "npm:quick-lru"; 3 3 import { createHash } from "node:crypto"; 4 4 - import { slingshoturl, constellationurl } from "../main.ts"; 4 4 + import { config } from "../config.ts"; 5 5 import * as ATPAPI from "npm:@atproto/api"; 6 6 7 7 const cache = new QuickLRU({ maxSize: 10000 }); ··· 57 57 export async function resolveIdentity( 58 58 actor: string 59 59 ): Promise<SlingshotMiniDoc> { 60 60 - const url = `${slingshoturl}/xrpc/com.bad-example.identity.resolveMiniDoc?identifier=${actor}`; 60 60 + const url = `${config.slingshot}/xrpc/com.bad-example.identity.resolveMiniDoc?identifier=${actor}`; 61 61 return (await cachedFetch(url)) as SlingshotMiniDoc; 62 62 } 63 63 export async function getRecord({ ··· 155 155 collection: string, 156 156 rkey: string 157 157 ): Promise<GetRecord> { 158 158 - const url = `${slingshoturl}/xrpc/com.atproto.repo.getRecord?repo=${did}&collection=${collection}&rkey=${rkey}`; 158 158 + const url = `${config.slingshot}/xrpc/com.atproto.repo.getRecord?repo=${did}&collection=${collection}&rkey=${rkey}`; 159 159 const result = (await cachedFetch(url)) as GetRecord; 160 160 return result as GetRecord; 161 161 } ··· 169 169 collection: string; 170 170 path: string; 171 171 }): Promise<number> { 172 172 - const url = `${constellationurl}/links/count/distinct-dids?target=${did}&collection=${collection}&path=${path}`; 172 172 + const url = `${config.constellation}/links/count/distinct-dids?target=${did}&collection=${collection}&path=${path}`; 173 173 const result = (await cachedFetch(url)) as ConstellationDistinctDids; 174 174 return result.total; 175 175 }