whey.party
an attempt to make a lightweight, easily self-hostable, scoped bluesky appview
i swear this is the final index server/view server split
-20
.env
-20
.env
···
1
-
# main indexers
2
-
JETSTREAM_URL="wss://jetstream1.us-east.bsky.network"
3
-
SPACEDUST_URL="wss://spacedust.whey.party"
4
-
5
-
# for backfill (useless if you just started the instance right now)
6
-
CONSTELLATION_URL="https://constellation.microcosm.blue"
7
-
# i dont actually know why i need this
8
-
SLINGSHOT_URL="https://slingshot.whey.party"
9
-
10
-
# bools
11
-
INDEX_SERVER_ENABLED=true
12
-
INDEX_SERVER_INVITES_REQUIRED=true
13
-
14
-
VIEW_SERVER_ENABLED=true
15
-
INDEX_SERVER_INVITES_REQUIRED=true
16
-
17
-
# this is for both index and view server btw
18
-
SERVICE_DID="did:web:local3768forumtest.whey.party"
19
-
SERVICE_ENDPOINT="https://local3768forumtest.whey.party"
20
-
SERVER_PORT="3768"
···
+3
-1
.gitignore
+3
-1
.gitignore
+35
config.jsonc.example
+35
config.jsonc.example
···
···
1
+
{
2
+
// Main indexers
3
+
"jetstream": "wss://jetstream1.us-east.bsky.network", // you can self host it -> https://github.com/bluesky-social/jetstream
4
+
"spacedust": "wss://spacedust.your.site", // you can self host it -> https://www.microcosm.blue
5
+
6
+
// For backfill (optional)
7
+
"constellation": "https://constellation.microcosm.blue", // (not useful on a new setup — requires pre-existing data to backfill)
8
+
9
+
// Utility services
10
+
"slingshot": "https://slingshot.your.site", // you can self host it -> https://www.microcosm.blue
11
+
12
+
// Index Server config
13
+
"indexServer": {
14
+
"inviteOnly": true,
15
+
"port": 3767,
16
+
"did": "did:web:skyliteindexserver.your.site", // should be the same domain as the endpoint
17
+
"host": "https://skyliteindexserver.your.site"
18
+
},
19
+
20
+
// View Server config
21
+
"viewServer": {
22
+
"inviteOnly": true,
23
+
"port": 3768,
24
+
"did": "did:web:skyliteviewserver.your.site", // should be the same domain as the endpoint
25
+
"host": "https://skyliteviewserver.your.site",
26
+
27
+
// In order of which skylite index servers or bsky appviews to use first
28
+
"indexPriority": [
29
+
"user#skylite_index", // user resolved skylite index server
30
+
"did:web:backupindexserver.your.site#skylite_index", // a specific skylite index server
31
+
"user#bsky_appview", // user resolved bsky appview
32
+
"did:web:api.bsky.app#bsky_appview" // a specific bsky appview
33
+
]
34
+
}
35
+
}
+45
config.ts
+45
config.ts
···
···
1
+
import { parse } from "jsr:@std/jsonc";
2
+
import * as z from "npm:zod";
3
+
4
+
// configure these from the config.jsonc file (you can use config.jsonc.example as reference)
5
+
const indexTarget = z.string().refine(
6
+
(val) => {
7
+
const parts = val.split("#");
8
+
if (parts.length !== 2) return false;
9
+
10
+
const [prefix, suffix] = parts;
11
+
const validPrefix = prefix === "user" || prefix.startsWith("did:web:");
12
+
const validSuffix = suffix === "skylite_index" || suffix === "bsky_appview";
13
+
14
+
return validPrefix && validSuffix;
15
+
},
16
+
{
17
+
message:
18
+
"Each indexPriority entry must be in the form 'user#skylite_index', 'user#bsky_appview', 'did:web:...#skylite_index', or 'did:web:...#bsky_appview'",
19
+
}
20
+
);
21
+
22
+
const ConfigSchema = z.object({
23
+
jetstream: z.string(),
24
+
spacedust: z.string(),
25
+
constellation: z.string(),
26
+
slingshot: z.string(),
27
+
indexServer: z.object({
28
+
inviteOnly: z.boolean(),
29
+
port: z.number(),
30
+
did: z.string(),
31
+
host: z.string(),
32
+
}),
33
+
viewServer: z.object({
34
+
inviteOnly: z.boolean(),
35
+
port: z.number(),
36
+
did: z.string(),
37
+
host: z.string(),
38
+
indexPriority: z.array(indexTarget),
39
+
}),
40
+
});
41
+
42
+
const raw = await Deno.readTextFile("config.jsonc");
43
+
const config = ConfigSchema.parse(parse(raw));
44
+
45
+
export { config };
+2
-1
deno.json
+2
-1
deno.json
+2
-2
index/jetstream.ts
+2
-2
index/jetstream.ts
···
1
import { Database } from "jsr:@db/sqlite@0.11";
2
-
import { handleIndex } from "../main.ts";
3
import { resolveRecordFromURI } from "../utils/records.ts";
4
import { JetstreamManager } from "../utils/sharders.ts";
5
···
29
});
30
}
31
32
-
export async function handleJetstream(msg: any) {
33
console.log("Received Jetstream message: ", msg);
34
35
const op = msg.commit.operation;
···
1
import { Database } from "jsr:@db/sqlite@0.11";
2
+
import { config } from "../config.ts";
3
import { resolveRecordFromURI } from "../utils/records.ts";
4
import { JetstreamManager } from "../utils/sharders.ts";
5
···
29
});
30
}
31
32
+
export async function handleJetstream(msg: any, handleIndex: Function) {
33
console.log("Received Jetstream message: ", msg);
34
35
const op = msg.commit.operation;
+6
-4
index/onboardingBackfill.ts
+6
-4
index/onboardingBackfill.ts
···
1
-
import { indexServerIndexer } from "../indexserver.ts";
2
-
import { systemDB } from "../main.ts"
3
import { FINEPDSAndHandleFromDid } from "../utils/identity.ts";
4
5
···
68
const doer = did;
69
const rev = undefined;
70
const aturi = uri;
71
72
-
indexServerIndexer({
73
op,
74
doer,
75
rev,
76
aturi,
77
value,
78
indexsrc: "onboarding_backfill",
79
-
userdbname: did,
80
})
81
return;
82
// console.log(`[BACKFILL] ${collection} -> ${uri}`);
···
1
+
import { genericIndexServer } from "../main-index.ts";
2
+
import { config } from "../config.ts"
3
import { FINEPDSAndHandleFromDid } from "../utils/identity.ts";
4
5
···
68
const doer = did;
69
const rev = undefined;
70
const aturi = uri;
71
+
const db = genericIndexServer.userManager.getDbForDid(doer);
72
+
if (!db) return;
73
74
+
genericIndexServer.indexServerIndexer({
75
op,
76
doer,
77
rev,
78
aturi,
79
value,
80
indexsrc: "onboarding_backfill",
81
+
db: db,
82
})
83
return;
84
// console.log(`[BACKFILL] ${collection} -> ${uri}`);
+1
-1
index/spacedust.ts
+1
-1
index/spacedust.ts
+56
-10
indexserver.ts
+56
-10
indexserver.ts
···
5
import * as IndexServerTypes from "./utils/indexservertypes.ts";
6
import { Database } from "jsr:@db/sqlite@0.11";
7
import { setupUserDb } from "./utils/dbuser.ts";
8
-
// import { systemDB } from "./main.ts";
9
import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts";
10
import { handleSpacedust, SpacedustLinkMessage } from "./index/spacedust.ts";
11
import { handleJetstream } from "./index/jetstream.ts";
12
import * as ATPAPI from "npm:@atproto/api";
13
import { AtUri } from "npm:@atproto/api";
14
import * as IndexServerAPI from "./indexclient/index.ts";
15
16
export interface IndexServerConfig {
17
baseDbPath: string;
···
77
return this.constellationAPIHandler(req);
78
}
79
return new Response("Not Found", { status: 404 });
80
}
81
82
// We will move all the global functions into this class as methods...
···
268
269
// TODO: not partial yet, currently skips refs
270
271
-
const qresult = this.queryActorLikes(jsonTyped.actor, jsonTyped.cursor);
272
if (!qresult) {
273
return new Response(
274
JSON.stringify({
···
1027
1028
return post;
1029
}
1030
queryFeedViewPost(
1031
uri: string
1032
): ATPAPI.AppBskyFeedDefs.FeedViewPost | undefined {
···
1041
};
1042
1043
return feedviewpost;
1044
}
1045
1046
// user feedgens
···
1213
return { items: [], cursor: undefined };
1214
}
1215
1216
-
queryActorLikes(
1217
did: string,
1218
cursor?: string
1219
):
1220
| {
1221
-
items: ATPAPI.AppBskyFeedDefs.FeedViewPost[];
1222
cursor: string | undefined;
1223
}
1224
| undefined {
1225
if (!this.isRegisteredIndexUser(did)) return;
1226
const db = this.userManager.getDbForDid(did);
1227
if (!db) return;
···
1249
}[];
1250
1251
const items = rows
1252
-
.map((row) => this.queryFeedViewPost(row.subject))
1253
-
.filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p);
1254
1255
const lastItem = rows[rows.length - 1];
1256
const nextCursor = lastItem
···
1418
1419
return { thread: returned };
1420
}
1421
-
1422
/**
1423
* please do not use this, use openDbForDid() instead
1424
* @param did
···
1431
//await db.exec(/* CREATE IF NOT EXISTS statements */);
1432
return db;
1433
}
1434
-
1435
isRegisteredIndexUser(did: string): boolean {
1436
const stmt = this.systemDB.prepare(`
1437
SELECT 1
···
1453
this.indexServer = indexServer;
1454
}
1455
1456
-
private users = new Map<string, UserIndexServer>();
1457
1458
/*async*/ addUser(did: string) {
1459
if (this.users.has(did)) return;
···
1508
constructor(indexServerUserManager: IndexServerUserManager, did: string) {
1509
this.did = did;
1510
this.indexServerUserManager = indexServerUserManager;
1511
-
this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid(this.did);
1512
// should probably put the params of exactly what were listening to here
1513
this.jetstream = new JetstreamManager((msg) => {
1514
console.log("Received Jetstream message: ", msg);
···
5
import * as IndexServerTypes from "./utils/indexservertypes.ts";
6
import { Database } from "jsr:@db/sqlite@0.11";
7
import { setupUserDb } from "./utils/dbuser.ts";
8
+
// import { systemDB } from "./env.ts";
9
import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts";
10
import { handleSpacedust, SpacedustLinkMessage } from "./index/spacedust.ts";
11
import { handleJetstream } from "./index/jetstream.ts";
12
import * as ATPAPI from "npm:@atproto/api";
13
import { AtUri } from "npm:@atproto/api";
14
import * as IndexServerAPI from "./indexclient/index.ts";
15
+
import * as IndexServerUtils from "./indexclient/util.ts"
16
17
export interface IndexServerConfig {
18
baseDbPath: string;
···
78
return this.constellationAPIHandler(req);
79
}
80
return new Response("Not Found", { status: 404 });
81
+
}
82
+
83
+
public handlesDid(did: string): boolean {
84
+
return this.userManager.handlesDid(did);
85
}
86
87
// We will move all the global functions into this class as methods...
···
273
274
// TODO: not partial yet, currently skips refs
275
276
+
const qresult = this.queryActorLikesPartial(jsonTyped.actor, jsonTyped.cursor);
277
if (!qresult) {
278
return new Response(
279
JSON.stringify({
···
1032
1033
return post;
1034
}
1035
+
1036
+
constructPostViewRef(uri: string): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef {
1037
+
const post: IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef = {
1038
+
uri: uri,
1039
+
cid: "cid.invalid", // oh shit we dont know the cid TODO: major design flaw
1040
+
};
1041
+
1042
+
return post;
1043
+
}
1044
+
1045
queryFeedViewPost(
1046
uri: string
1047
): ATPAPI.AppBskyFeedDefs.FeedViewPost | undefined {
···
1056
};
1057
1058
return feedviewpost;
1059
+
}
1060
+
1061
+
constructFeedViewPostRef(
1062
+
uri: string
1063
+
): IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef {
1064
+
const post = this.constructPostViewRef(uri);
1065
+
1066
+
const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = {
1067
+
$type: "party.whey.app.bsky.feed.defs#feedViewPostRef",
1068
+
post: post as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>,
1069
+
}
1070
+
1071
+
return feedviewpostref
1072
}
1073
1074
// user feedgens
···
1241
return { items: [], cursor: undefined };
1242
}
1243
1244
+
queryActorLikesPartial(
1245
did: string,
1246
cursor?: string
1247
):
1248
| {
1249
+
items: (ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef)[];
1250
cursor: string | undefined;
1251
}
1252
| undefined {
1253
+
// early return only if the actor did is not registered
1254
if (!this.isRegisteredIndexUser(did)) return;
1255
const db = this.userManager.getDbForDid(did);
1256
if (!db) return;
···
1278
}[];
1279
1280
const items = rows
1281
+
.map((row) => {
1282
+
const subjectDid = new AtUri(row.subject).host;
1283
+
1284
+
if (this.handlesDid(subjectDid)) {
1285
+
return this.queryFeedViewPost(row.subject);
1286
+
} else {
1287
+
return this.constructFeedViewPostRef(row.subject);
1288
+
}
1289
+
})
1290
+
.filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef => !!p);
1291
1292
const lastItem = rows[rows.length - 1];
1293
const nextCursor = lastItem
···
1455
1456
return { thread: returned };
1457
}
1458
+
1459
/**
1460
* please do not use this, use openDbForDid() instead
1461
* @param did
···
1468
//await db.exec(/* CREATE IF NOT EXISTS statements */);
1469
return db;
1470
}
1471
+
/**
1472
+
* @deprecated use handlesDid() instead
1473
+
* @param did
1474
+
* @returns
1475
+
*/
1476
isRegisteredIndexUser(did: string): boolean {
1477
const stmt = this.systemDB.prepare(`
1478
SELECT 1
···
1494
this.indexServer = indexServer;
1495
}
1496
1497
+
public users = new Map<string, UserIndexServer>();
1498
+
public handlesDid(did: string): boolean {
1499
+
return this.users.has(did);
1500
+
}
1501
1502
/*async*/ addUser(did: string) {
1503
if (this.users.has(did)) return;
···
1552
constructor(indexServerUserManager: IndexServerUserManager, did: string) {
1553
this.did = did;
1554
this.indexServerUserManager = indexServerUserManager;
1555
+
this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid(
1556
+
this.did
1557
+
);
1558
// should probably put the params of exactly what were listening to here
1559
this.jetstream = new JetstreamManager((msg) => {
1560
console.log("Received Jetstream message: ", msg);
+125
main-index.ts
+125
main-index.ts
···
···
1
+
import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts";
2
+
import { setupSystemDb } from "./utils/dbsystem.ts";
3
+
import { didDocument } from "./utils/diddoc.ts";
4
+
import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts";
5
+
import { IndexServer, IndexServerConfig } from "./indexserver.ts"
6
+
import { extractDid } from "./utils/identity.ts";
7
+
import { config } from "./config.ts";
8
+
9
+
// ------------------------------------------
10
+
// AppView Setup
11
+
// ------------------------------------------
12
+
13
+
const indexServerConfig: IndexServerConfig = {
14
+
baseDbPath: './dbs/registered-users', // The directory for user databases
15
+
systemDbPath: './dbs/registered-users/system.db', // The path for the main system database
16
+
jetstreamUrl: config.jetstream
17
+
};
18
+
export const genericIndexServer = new IndexServer(indexServerConfig);
19
+
setupSystemDb(genericIndexServer.systemDB);
20
+
21
+
// add me lol
22
+
genericIndexServer.systemDB.exec(`
23
+
INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus)
24
+
VALUES (
25
+
'did:plc:mn45tewwnse5btfftvd3powc',
26
+
'admin',
27
+
datetime('now'),
28
+
'ready'
29
+
);
30
+
31
+
INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus)
32
+
VALUES (
33
+
'did:web:did12.whey.party',
34
+
'admin',
35
+
datetime('now'),
36
+
'ready'
37
+
);
38
+
`)
39
+
40
+
genericIndexServer.start();
41
+
42
+
// ------------------------------------------
43
+
// XRPC Method Implementations
44
+
// ------------------------------------------
45
+
46
+
// const indexServerRoutes = new Set([
47
+
// "/xrpc/app.bsky.actor.getProfile",
48
+
// "/xrpc/app.bsky.actor.getProfiles",
49
+
// "/xrpc/app.bsky.feed.getActorFeeds",
50
+
// "/xrpc/app.bsky.feed.getFeedGenerator",
51
+
// "/xrpc/app.bsky.feed.getFeedGenerators",
52
+
// "/xrpc/app.bsky.feed.getPosts",
53
+
// "/xrpc/party.whey.app.bsky.feed.getActorLikesPartial",
54
+
// "/xrpc/party.whey.app.bsky.feed.getAuthorFeedPartial",
55
+
// "/xrpc/party.whey.app.bsky.feed.getLikesPartial",
56
+
// "/xrpc/party.whey.app.bsky.feed.getPostThreadPartial",
57
+
// "/xrpc/party.whey.app.bsky.feed.getQuotesPartial",
58
+
// "/xrpc/party.whey.app.bsky.feed.getRepostedByPartial",
59
+
// // more federated endpoints, not planned yet, lexicons will come later
60
+
// /*
61
+
// app.bsky.graph.getLists // doesnt need to because theres no items[], and its self ProfileViewBasic
62
+
// app.bsky.graph.getList // needs to be Partial-ed (items[] union with ProfileViewRef)
63
+
// app.bsky.graph.getActorStarterPacks // maybe doesnt need to be Partial-ed because its self ProfileViewBasic
64
+
65
+
// app.bsky.feed.getListFeed // uhh actually already exists its getListFeedPartial
66
+
// */
67
+
// "/xrpc/party.whey.app.bsky.feed.getListFeedPartial",
68
+
// ]);
69
+
70
+
Deno.serve(
71
+
{ port: config.indexServer.port },
72
+
(req: Request): Response => {
73
+
const url = new URL(req.url);
74
+
const pathname = url.pathname;
75
+
const searchParams = searchParamsToJson(url.searchParams);
76
+
77
+
if (pathname === "/.well-known/did.json") {
78
+
return new Response(JSON.stringify(didDocument("index",config.indexServer.did,config.indexServer.host,"whatever")), {
79
+
headers: withCors({ "Content-Type": "application/json" }),
80
+
});
81
+
}
82
+
if (pathname === "/health") {
83
+
return new Response("OK", {
84
+
status: 200,
85
+
headers: withCors({
86
+
"Content-Type": "text/plain",
87
+
}),
88
+
});
89
+
}
90
+
if (req.method === "OPTIONS") {
91
+
return new Response(null, {
92
+
status: 204,
93
+
headers: {
94
+
"Access-Control-Allow-Origin": "*",
95
+
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
96
+
"Access-Control-Allow-Headers": "*",
97
+
},
98
+
});
99
+
}
100
+
console.log(`request for "${pathname}"`)
101
+
const constellation = pathname.startsWith("/links")
102
+
103
+
if (constellation) {
104
+
const target = searchParams?.target as string
105
+
const safeDid = extractDid(target);
106
+
const targetserver = genericIndexServer.handlesDid(safeDid)
107
+
if (targetserver) {
108
+
return genericIndexServer.constellationAPIHandler(req);
109
+
} else {
110
+
return new Response(
111
+
JSON.stringify({
112
+
error: "User not found",
113
+
}),
114
+
{
115
+
status: 404,
116
+
headers: withCors({ "Content-Type": "application/json" }),
117
+
}
118
+
);
119
+
}
120
+
} else {
121
+
// indexServerRoutes.has(pathname)
122
+
return genericIndexServer.indexServerHandler(req);
123
+
}
124
+
}
125
+
);
+59
main-view.ts
+59
main-view.ts
···
···
1
+
import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts";
2
+
import { setupSystemDb } from "./utils/dbsystem.ts";
3
+
import { didDocument } from "./utils/diddoc.ts";
4
+
import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts";
5
+
import { IndexServer, IndexServerConfig } from "./indexserver.ts"
6
+
import { extractDid } from "./utils/identity.ts";
7
+
import { config } from "./config.ts";
8
+
import { viewServerHandler } from "./viewserver.ts";
9
+
10
+
// ------------------------------------------
11
+
// AppView Setup
12
+
// ------------------------------------------
13
+
14
+
setupAuth({
15
+
serviceDid: config.viewServer.did,
16
+
//keyCacheSize: 500,
17
+
//keyCacheTTL: 10 * 60 * 1000,
18
+
});
19
+
20
+
// ------------------------------------------
21
+
// XRPC Method Implementations
22
+
// ------------------------------------------
23
+
24
+
25
+
Deno.serve(
26
+
{ port: config.viewServer.port },
27
+
async (req: Request): Promise<Response> => {
28
+
const url = new URL(req.url);
29
+
const pathname = url.pathname;
30
+
const searchParams = searchParamsToJson(url.searchParams);
31
+
32
+
if (pathname === "/.well-known/did.json") {
33
+
return new Response(JSON.stringify(didDocument), {
34
+
headers: withCors({ "Content-Type": "application/json" }),
35
+
});
36
+
}
37
+
if (pathname === "/health") {
38
+
return new Response("OK", {
39
+
status: 200,
40
+
headers: withCors({
41
+
"Content-Type": "text/plain",
42
+
}),
43
+
});
44
+
}
45
+
if (req.method === "OPTIONS") {
46
+
return new Response(null, {
47
+
status: 204,
48
+
headers: {
49
+
"Access-Control-Allow-Origin": "*",
50
+
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
51
+
"Access-Control-Allow-Headers": "*",
52
+
},
53
+
});
54
+
}
55
+
console.log(`request for "${pathname}"`)
56
+
57
+
return await viewServerHandler(req)
58
+
}
59
+
);
-172
main.ts
-172
main.ts
···
1
-
import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts";
2
-
import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts";
3
-
import { resolveRecordFromURI, validateRecord } from "./utils/records.ts";
4
-
import { setupSystemDb } from "./utils/dbsystem.ts";
5
-
import { setupUserDb } from "./utils/dbuser.ts";
6
-
import { handleSpacedust, startSpacedust } from "./index/spacedust.ts";
7
-
import { handleJetstream, startJetstream } from "./index/jetstream.ts";
8
-
import { Database } from "jsr:@db/sqlite@0.11";
9
-
//import express from "npm:express";
10
-
//import { createServer } from "./xrpc/index.ts";
11
-
import { indexHandlerContext } from "./index/types.ts";
12
-
import * as IndexServerTypes from "./utils/indexservertypes.ts";
13
-
import * as ViewServerTypes from "./utils/viewservertypes.ts";
14
-
import * as ATPAPI from "npm:@atproto/api";
15
-
import { didDocument } from "./utils/diddoc.ts";
16
-
import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts";
17
-
import { IndexServer, IndexServerConfig } from "./indexserver.ts"
18
-
import { viewServerHandler } from "./viewserver.ts";
19
-
20
-
export const jetstreamurl = Deno.env.get("JETSTREAM_URL");
21
-
export const slingshoturl = Deno.env.get("SLINGSHOT_URL");
22
-
export const constellationurl = Deno.env.get("CONSTELLATION_URL");
23
-
export const spacedusturl = Deno.env.get("SPACEDUST_URL");
24
-
25
-
// ------------------------------------------
26
-
// AppView Setup
27
-
// ------------------------------------------
28
-
29
-
const config: IndexServerConfig = {
30
-
baseDbPath: './dbs', // The directory for user databases
31
-
systemDbPath: './system.db', // The path for the main system database
32
-
jetstreamUrl: jetstreamurl || ""
33
-
};
34
-
const registeredUsersIndexServer = new IndexServer(config);
35
-
setupSystemDb(registeredUsersIndexServer.systemDB);
36
-
37
-
// add me lol
38
-
registeredUsersIndexServer.systemDB.exec(`
39
-
INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus)
40
-
VALUES (
41
-
'did:plc:mn45tewwnse5btfftvd3powc',
42
-
'admin',
43
-
datetime('now'),
44
-
'ready'
45
-
);
46
-
47
-
INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus)
48
-
VALUES (
49
-
'did:web:did12.whey.party',
50
-
'admin',
51
-
datetime('now'),
52
-
'ready'
53
-
);
54
-
`)
55
-
56
-
registeredUsersIndexServer.start();
57
-
58
-
// should do both of these per user actually, since now each user has their own db
59
-
// also the set of records and backlinks to listen should be seperate between index and view servers
60
-
// damn
61
-
// export const spacedustManager = new SpacedustManager((msg) =>
62
-
// handleSpacedust(msg)
63
-
// );
64
-
// export const jetstreamManager = new JetstreamManager((msg) =>
65
-
// handleJetstream(msg)
66
-
// );
67
-
// startSpacedust();
68
-
// startJetstream();
69
-
70
-
// 1. connect to system db
71
-
// 2. get all registered users
72
-
// parse config (maybe some are only indexes and maybe some are only views)
73
-
// map all new jetstream and spacedust listeners
74
-
// call handleIndex with the specific db to use
75
-
76
-
setupAuth({
77
-
// local3768forumtest is just my tunnel from my dev env to the outside web that im reusing from forumtest
78
-
serviceDid: `${Deno.env.get("SERVICE_DID")}`,
79
-
//keyCacheSize: 500,
80
-
//keyCacheTTL: 10 * 60 * 1000,
81
-
});
82
-
83
-
// ------------------------------------------
84
-
// XRPC Method Implementations
85
-
// ------------------------------------------
86
-
87
-
const indexServerRoutes = new Set([
88
-
"/xrpc/app.bsky.actor.getProfile",
89
-
"/xrpc/app.bsky.actor.getProfiles",
90
-
"/xrpc/app.bsky.feed.getActorFeeds",
91
-
"/xrpc/app.bsky.feed.getFeedGenerator",
92
-
"/xrpc/app.bsky.feed.getFeedGenerators",
93
-
"/xrpc/app.bsky.feed.getPosts",
94
-
"/xrpc/party.whey.app.bsky.feed.getActorLikesPartial",
95
-
"/xrpc/party.whey.app.bsky.feed.getAuthorFeedPartial",
96
-
"/xrpc/party.whey.app.bsky.feed.getLikesPartial",
97
-
"/xrpc/party.whey.app.bsky.feed.getPostThreadPartial",
98
-
"/xrpc/party.whey.app.bsky.feed.getQuotesPartial",
99
-
"/xrpc/party.whey.app.bsky.feed.getRepostedByPartial",
100
-
// more federated endpoints, not planned yet, lexicons will come later
101
-
/*
102
-
app.bsky.graph.getLists // doesnt need to because theres no items[], and its self ProfileViewBasic
103
-
app.bsky.graph.getList // needs to be Partial-ed (items[] union with ProfileViewRef)
104
-
app.bsky.graph.getActorStarterPacks // maybe doesnt need to be Partial-ed because its self ProfileViewBasic
105
-
106
-
app.bsky.feed.getListFeed // uhh actually already exists its getListFeedPartial
107
-
*/
108
-
"/xrpc/party.whey.app.bsky.feed.getListFeedPartial",
109
-
]);
110
-
111
-
Deno.serve(
112
-
{ port: Number(`${Deno.env.get("SERVER_PORT")}`) },
113
-
async (req: Request): Promise<Response> => {
114
-
const url = new URL(req.url);
115
-
const pathname = url.pathname;
116
-
// const searchParams = searchParamsToJson(url.searchParams);
117
-
// let reqBody: undefined | string;
118
-
// let jsonbody: undefined | Record<string, unknown>;
119
-
// try {
120
-
// const clone = req.clone();
121
-
// jsonbody = await clone.json();
122
-
// } catch (e) {
123
-
// console.warn("Request body is not valid JSON:", e);
124
-
// }
125
-
if (pathname === "/.well-known/did.json") {
126
-
return new Response(JSON.stringify(didDocument), {
127
-
headers: withCors({ "Content-Type": "application/json" }),
128
-
});
129
-
}
130
-
if (pathname === "/health") {
131
-
return new Response("OK", {
132
-
status: 200,
133
-
headers: withCors({
134
-
"Content-Type": "text/plain",
135
-
}),
136
-
});
137
-
}
138
-
if (req.method === "OPTIONS") {
139
-
return new Response(null, {
140
-
status: 204,
141
-
headers: {
142
-
"Access-Control-Allow-Origin": "*",
143
-
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
144
-
"Access-Control-Allow-Headers": "*",
145
-
},
146
-
});
147
-
}
148
-
// const bskyUrl = `https://api.bsky.app${pathname}${url.search}`;
149
-
// const hasAuth = req.headers.has("authorization");
150
-
// const xrpcMethod = pathname.startsWith("/xrpc/")
151
-
// ? pathname.slice("/xrpc/".length)
152
-
// : null;
153
-
console.log(`request for "${pathname}"`)
154
-
const constellation = pathname.startsWith("/links")
155
-
156
-
// return await viewServerHandler(req)
157
-
158
-
if (constellation) {
159
-
return registeredUsersIndexServer.constellationAPIHandler(req);
160
-
}
161
-
162
-
if (indexServerRoutes.has(pathname)) {
163
-
return registeredUsersIndexServer.indexServerHandler(req);
164
-
} else {
165
-
return await viewServerHandler(req);
166
-
}
167
-
}
168
-
);
169
-
170
-
// ------------------------------------------
171
-
// Indexer
172
-
// ------------------------------------------
···
+2
-2
readme.md
+2
-2
readme.md
···
44
this project is pre-alpha and not intended for general use yet. you are welcome to experiment if you dont mind errors or breaking changes.
45
46
the project is split into two, the "Index Server" and the "View Server".
47
-
these currently run in a single process and share the same HTTP server and port.
48
49
-
configuration is in the `.env` file
50
51
expose your localhost to the web using a tunnel or something and use that url as the custom appview url
52
···
44
this project is pre-alpha and not intended for general use yet. you are welcome to experiment if you dont mind errors or breaking changes.
45
46
the project is split into two, the "Index Server" and the "View Server".
47
+
despite both living in this repo, they run different http servers with different configs
48
49
+
example configuration is in the `config.jsonc.example` file
50
51
expose your localhost to the web using a tunnel or something and use that url as the custom appview url
52
+37
-22
utils/diddoc.ts
+37
-22
utils/diddoc.ts
···
1
-
export const didDocument = {
2
-
"@context": [
3
-
"https://www.w3.org/ns/did/v1",
4
-
"https://w3id.org/security/multikey/v1",
5
-
],
6
-
id: `${Deno.env.get("SERVICE_DID")}`,
7
-
verificationMethod: [
8
-
{
9
-
id: `${Deno.env.get("SERVICE_DID")}#atproto`,
10
-
type: "Multikey",
11
-
controller: `${Deno.env.get("SERVICE_DID")}`,
12
-
publicKeyMultibase: "bullshit",
13
},
14
-
],
15
-
service: [
16
-
{
17
id: "#bsky_notif",
18
type: "BskyNotificationService",
19
-
serviceEndpoint: `${Deno.env.get("SERVICE_ENDPOINT")}`,
20
},
21
-
{
22
-
id: "#bsky_appview",
23
-
type: "BskyAppView",
24
-
serviceEndpoint: `${Deno.env.get("SERVICE_ENDPOINT")}`,
25
},
26
-
],
27
-
};
···
1
+
// type "both" should not be used
2
+
export function didDocument(
3
+
type: "view" | "index" | "both",
4
+
did: string,
5
+
endpoint: string,
6
+
publicKeyMultibase: string,
7
+
) {
8
+
const services = [
9
+
(type === "view" || type === "both") && {
10
+
id: "#bsky_appview",
11
+
type: "BskyAppView",
12
+
serviceEndpoint: endpoint,
13
},
14
+
(type === "view" || type === "both") && {
15
id: "#bsky_notif",
16
type: "BskyNotificationService",
17
+
serviceEndpoint: endpoint,
18
},
19
+
(type === "index" || type === "both") && {
20
+
id: "#skylite_index",
21
+
type: "SkyliteIndexServer",
22
+
serviceEndpoint: endpoint,
23
},
24
+
].filter(Boolean);
25
+
26
+
return {
27
+
"@context": [
28
+
"https://www.w3.org/ns/did/v1",
29
+
"https://w3id.org/security/multikey/v1",
30
+
],
31
+
id: did,
32
+
verificationMethod: [
33
+
{
34
+
id: `${did}#atproto`,
35
+
type: "Multikey",
36
+
controller: did,
37
+
publicKeyMultibase: publicKeyMultibase,
38
+
},
39
+
],
40
+
service: services,
41
+
};
42
+
}
+14
utils/identity.ts
+14
utils/identity.ts
···
1
2
import { DidResolver, HandleResolver } from "npm:@atproto/identity";
3
import { Database } from "jsr:@db/sqlite@0.11";
4
const systemDB = new Database("./system.db") // TODO: temporary shim. should seperate this to its own central system db instead of the now instantiated system dbs
5
type DidMethod = "web" | "plc";
6
type DidDoc = {
···
224
} catch (err) {
225
console.error(`Failed to extract/store PDS and handle for '${did}':`, err);
226
return null;
227
}
228
}
···
1
2
import { DidResolver, HandleResolver } from "npm:@atproto/identity";
3
import { Database } from "jsr:@db/sqlite@0.11";
4
+
import { AtUri } from "npm:@atproto/api";
5
const systemDB = new Database("./system.db") // TODO: temporary shim. should seperate this to its own central system db instead of the now instantiated system dbs
6
type DidMethod = "web" | "plc";
7
type DidDoc = {
···
225
} catch (err) {
226
console.error(`Failed to extract/store PDS and handle for '${did}':`, err);
227
return null;
228
+
}
229
+
}
230
+
231
+
export function extractDid(input: string): string {
232
+
if (input.startsWith('did:')) {
233
+
return input
234
+
}
235
+
236
+
try {
237
+
const uri = new AtUri(input)
238
+
return uri.host
239
+
} catch (e) {
240
+
throw new Error(`Invalid input: expected a DID or a valid AT URI, got "${input}"`)
241
}
242
}
+4
-4
utils/server.ts
+4
-4
utils/server.ts
···
1
import ky from "npm:ky";
2
import QuickLRU from "npm:quick-lru";
3
import { createHash } from "node:crypto";
4
-
import { slingshoturl, constellationurl } from "../main.ts";
5
import * as ATPAPI from "npm:@atproto/api";
6
7
const cache = new QuickLRU({ maxSize: 10000 });
···
57
export async function resolveIdentity(
58
actor: string
59
): Promise<SlingshotMiniDoc> {
60
-
const url = `${slingshoturl}/xrpc/com.bad-example.identity.resolveMiniDoc?identifier=${actor}`;
61
return (await cachedFetch(url)) as SlingshotMiniDoc;
62
}
63
export async function getRecord({
···
155
collection: string,
156
rkey: string
157
): Promise<GetRecord> {
158
-
const url = `${slingshoturl}/xrpc/com.atproto.repo.getRecord?repo=${did}&collection=${collection}&rkey=${rkey}`;
159
const result = (await cachedFetch(url)) as GetRecord;
160
return result as GetRecord;
161
}
···
169
collection: string;
170
path: string;
171
}): Promise<number> {
172
-
const url = `${constellationurl}/links/count/distinct-dids?target=${did}&collection=${collection}&path=${path}`;
173
const result = (await cachedFetch(url)) as ConstellationDistinctDids;
174
return result.total;
175
}
···
1
import ky from "npm:ky";
2
import QuickLRU from "npm:quick-lru";
3
import { createHash } from "node:crypto";
4
+
import { config } from "../config.ts";
5
import * as ATPAPI from "npm:@atproto/api";
6
7
const cache = new QuickLRU({ maxSize: 10000 });
···
57
export async function resolveIdentity(
58
actor: string
59
): Promise<SlingshotMiniDoc> {
60
+
const url = `${config.slingshot}/xrpc/com.bad-example.identity.resolveMiniDoc?identifier=${actor}`;
61
return (await cachedFetch(url)) as SlingshotMiniDoc;
62
}
63
export async function getRecord({
···
155
collection: string,
156
rkey: string
157
): Promise<GetRecord> {
158
+
const url = `${config.slingshot}/xrpc/com.atproto.repo.getRecord?repo=${did}&collection=${collection}&rkey=${rkey}`;
159
const result = (await cachedFetch(url)) as GetRecord;
160
return result as GetRecord;
161
}
···
169
collection: string;
170
path: string;
171
}): Promise<number> {
172
+
const url = `${config.constellation}/links/count/distinct-dids?target=${did}&collection=${collection}&path=${path}`;
173
const result = (await cachedFetch(url)) as ConstellationDistinctDids;
174
return result.total;
175
}