futur.blue
objective categorical abstract machine language personal data server
Document env vars in .env.example
+59
.env.example
+59
.env.example
···
1
+
# (optional) debug, info, warn, or error
2
+
# default: info
3
+
# PDS_LOG_LEVEL=info
4
+
5
+
# (optional) directory where databases and blobs will be stored
6
+
# default: ./data [relative to cwd]
7
+
# PDS_DATA_DIR=./data
8
+
9
+
# (required) hostname where this PDS is running
10
+
# for handles to work, you should set A records for *.{PDS_HOSTNAME} and PDS_HOSTNAME to point to this server
11
+
PDS_HOSTNAME=
12
+
13
+
# (optional) the PDS's DID
14
+
# default: did:web:{PDS_HOSTNAME}
15
+
# PDS_DID=
16
+
17
+
# (optional) whether an invite code is needed to create an account
18
+
# default: true
19
+
# PDS_INVITE_CODE_REQUIRED=true
20
+
21
+
# (required) the rotation key the PDS will use, in multibase format
22
+
# tip: run the `gen-keys` binary to generate this (more info in readme)
23
+
PDS_ROTATION_KEY=
24
+
25
+
# (required) the key used to sign JWKs
26
+
# tip: run the `gen-keys` binary to generate this (more info in readme)
27
+
PDS_JWK_MULTIBASE=
28
+
29
+
# (required) the PDS admin password
30
+
# this can be used to access com.atproto.admin.* endpoints as well as the /admin dashboard
31
+
PDS_ADMIN_PASSWORD=
32
+
33
+
# (optional) relays/crawlers to inform to subscribe to this PDS, comma-separated
34
+
# default: https://bsky.network
35
+
# PDS_CRAWLERS=https://bsky.network
36
+
37
+
# (optional [but recommended]): a secret token used to generate DPoP nonces; 32 bytes, base64url
38
+
# tip: run the `gen-keys` binary to generate this (more info in readme)
39
+
# PDS_DPOP_NONCE_SECRET=
40
+
41
+
# (optional) email config (see readme for details)
42
+
# auth uri should look like smtp[s]://user:pass@host[:port]
43
+
# sender should look like e@mail.com or Name <e@mail.com>
44
+
# without setting these, emails will be logged to stdout
45
+
# PDS_SMTP_STARTTLS=false
46
+
# PDS_SMTP_AUTH_URI=
47
+
# PDS_SMTP_SENDER=
48
+
49
+
# (optional) S3 config (see readme for details)
50
+
# set these to back up your databases and/or blobs to S3(-compatible storge)
51
+
# PDS_S3_BLOBS_ENABLED=false
52
+
# PDS_S3_BACKUPS_ENABLED=false
53
+
# PDS_S3_BACKUP_INTERVAL_S=3600
54
+
# PDS_S3_ENDPOINT=
55
+
# PDS_S3_REGION=
56
+
# PDS_S3_BUCKET=
57
+
# PDS_S3_ACCESS_KEY=
58
+
# PDS_S3_SECRET_KEY=
59
+
# PDS_S3_CDN_URL=
+25
-11
README.md
+25
-11
README.md
···
18
18
19
19
to generate some of the environment variables you'll need.
20
20
21
-
Head to [`docker-compose.yaml`](docker-compose.yaml) and fill in the environment variables marked as required. See [Environment](#environment) for further details on configuration.
21
+
Copy [`.env.example`](.env.example) to `.env` and fill in the environment variables marked as required. See [Environment](#environment) for further details on configuration.
22
22
23
23
After that, run
24
+
24
25
```
25
26
docker compose up -d
26
27
```
28
+
27
29
to start the PDS, then navigate to `https://{PDS_HOSTNAME}/admin` to log in with the admin password you specified and create an invite code or a new account on the PDS.
28
30
29
31
## environment
30
32
31
-
Documentation for most environment variables can be found in [`docker-compose.yaml`](docker-compose.yaml). There are two optional categories of environment variables that add functionality:
33
+
Documentation for most environment variables can be found in [`.env.example`](.env.example). There are two optional categories of environment variables that add functionality:
32
34
33
35
### SMTP
36
+
34
37
The PDS can email users for password changes, identity updates, and account deletion. If these environment variables are not set, emails will instead be logged to the process' stdout.
35
38
36
39
- `PDS_SMTP_AUTH_URI` — The URI to connect to the mail server. This should look like `smtp[s]://user:pass@host[:port]`.
···
38
41
- `PDS_SMTP_SENDER` — The identity to send emails as. Can be an email address (`e@mail.com`) or a mailbox (`Name <e@mail.com>`).
39
42
40
43
### S3
44
+
41
45
The PDS can be configured to back up server data to and/or store blobs in S3(-compatible storage).
42
46
43
47
- `PDS_S3_BLOBS_ENABLED=false` — Whether to store blobs in S3. By default, blobs are stored locally in `{PDS_DATA_DIR}/blobs/[did]/`.
44
-
- `PDS_S3_BACKUPS_ENABLED=false` — Whether to back up data to S3.
48
+
- `PDS_S3_BACKUPS_ENABLED=false` — Whether to back up data to S3.
45
49
- `PDS_S3_BACKUP_INTERVAL_S=3600` — How often to back up to S3, in seconds.
46
50
- `PDS_S3_ENDPOINT`, `PDS_S3_REGION`, `PDS_S3_BUCKET`, `PDS_S3_ACCESS_KEY`, `PDS_S3_SECRET_KEY` — S3 configuration.
47
51
- `PDS_S3_CDN_URL` — You may optionally set this to redirect `getBlob` requests to `{PDS_S3_CDN_URL}/blobs/{did}/{cid}`. When unset, blobs will be fetched either from local storage or from S3, depending on `PDS_S3_BLOBS_ENABLED`.
···
50
54
51
55
This repo contains several libraries in addition to the `pegasus` PDS:
52
56
53
-
| library | description |
54
-
|----------|-------------|
55
-
| frontend | The PDS frontend, containing the admin dashboard and account page. |
57
+
| library | description |
58
+
| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
59
+
| frontend | The PDS frontend, containing the admin dashboard and account page. |
56
60
| ipld | A mostly [DASL-compliant](https://dasl.ing/) implementation of [CIDs](https://dasl.ing/cid.html), [CAR](https://dasl.ing/car.html), and [DAG-CBOR](https://dasl.ing/drisl.html). |
57
-
| kleidos | An atproto-valid interface for secp256k1 and secp256r1 key management, signing/verifying, and encoding/decoding. |
58
-
| mist | A [Merkle Search Tree](https://atproto.com/specs/repository#mst-structure) implementation for data repository purposes. |
59
-
| pegasus | The PDS implementation. |
61
+
| kleidos | An atproto-valid interface for secp256k1 and secp256r1 key management, signing/verifying, and encoding/decoding. |
62
+
| mist | A [Merkle Search Tree](https://atproto.com/specs/repository#mst-structure) implementation for data repository purposes. |
63
+
| pegasus | The PDS implementation. |
60
64
61
65
To start developing, you'll need:
66
+
62
67
- [`opam`](https://opam.ocaml.org/doc/Install.html), the OCaml Package Manager
63
68
- and the following packages, or their equivalents on your operating system: `cmake git libev-dev libffi-dev libgmp-dev libssl-dev libsqlite3-dev libpcre3-dev pkg-config`
64
69
65
70
Start by creating an opam switch; similar to a Python virtual environment, storing the dependencies for this project and a specific compiler version. After that, install [`dune`](https://dune.build), the build system/package manager pegasus uses.
71
+
66
72
```
67
73
opam switch create . 5.2.1
68
74
opam install dune
69
75
```
70
76
71
77
You may need to run `eval $(opam env)` for this to work. Next, run
78
+
72
79
```
73
80
dune pkg lock
74
81
```
82
+
75
83
to solve dependencies.
76
84
77
-
Finally, after setting the required environment variables (see [Environment](#environment)), either run
85
+
Set the required environment variables (see [Environment](#environment)), noting that the program won't automatically read from `.env`, then either run
86
+
78
87
```
79
88
dune exec pegasus
80
89
```
90
+
81
91
to run the program directly, or
92
+
82
93
```
83
94
dune build
84
95
```
96
+
85
97
to produce an executable that you'll likely find in `_build/default/bin`.
86
98
87
99
For development, you'll also want to run
100
+
88
101
```
89
102
dune tools exec ocamlformat
90
103
dune tools exec ocamllsp
91
104
```
105
+
92
106
to download the formatter and LSP services. You can run `dune fmt` to format the project.
93
107
94
-
The [frontend](frontend) is written in [MLX](https://github.com/ocaml-mlx/mlx), a JSX-ish OCaml dialect. To format it, you'll need to `opam install ocamlformat-mlx`, then `ocamlformat-mlx -i frontend/**/*.mlx`. You'll see a few errors on formatting files containing `[%browser_only]`; I'm waiting on the next release of `mlx` to fix those.
108
+
The [frontend](frontend) is written in [MLX](https://github.com/ocaml-mlx/mlx), a JSX-ish OCaml dialect. To format it, you'll need to `opam install ocamlformat-mlx`, then `ocamlformat-mlx -i frontend/**/*.mlx`. You'll see a few errors on formatting files containing `[%browser_only]`; I'm waiting on the next release of `mlx` to fix those.
+23
-59
docker-compose.yaml
+23
-59
docker-compose.yaml
···
7
7
volumes:
8
8
- pds:/data
9
9
environment:
10
-
# (optional) debug, info, warn, or error
11
-
# default: info
12
-
# - PDS_LOG_LEVEL=info
13
-
14
-
# (optional) directory where databases and blobs will be stored
15
-
# default: ./data [relative to cwd]
16
-
- PDS_DATA_DIR=./data
17
-
18
-
# (required) hostname where this PDS is running
19
-
# for handles to work, you should configure *.hostname and hostname to point to this server
20
-
# make sure to also set this env var for the caddy container below
21
-
- PDS_HOSTNAME=
22
-
23
-
# (optional) the PDS's DID
24
-
# default: did:web:{hostname}
25
-
# - PDS_DID=
26
-
27
-
# (optional) whether an invite code is needed to create an account
28
-
# default: true
29
-
# - PDS_INVITE_CODE_REQUIRED=true
30
-
31
-
# (required) the rotation key the PDS will use, in multibase format
32
-
# tip: run the `gen-keys` binary to generate this
33
-
- PDS_ROTATION_KEY=
34
-
35
-
# (required) the key used to sign JWKs
36
-
# tip: run the `gen-keys` binary to generate this
37
-
- PDS_JWK_MULTIBASE=
38
-
39
-
# (required) the PDS admin password
40
-
# this can be used to access com.atproto.admin.* endpoints as well as the /admin dashboard
41
-
- PDS_ADMIN_PASSWORD=
42
-
43
-
# (optional) relays/crawlers to inform to subscribe to this PDS, comma-separated
44
-
# default: https://bsky.network
45
-
# - PDS_CRAWLERS=https://bsky.network
46
-
47
-
# (optional [but recommended]): a secret token used to generate DPoP nonces; 32 bytes, base64url
48
-
# tip: run the `gen-keys` binary to generate this
49
-
# - PDS_DPOP_NONCE_SECRET=
10
+
- PDS_LOG_LEVEL=${PDS_LOG_LEVEL:-info}
11
+
- PDS_DATA_DIR=${PDS_DATA_DIR:-./data}
12
+
- PDS_HOSTNAME=${PDS_HOSTNAME:?}
13
+
- PDS_DID=${PDS_DID}
14
+
- PDS_INVITE_CODE_REQUIRED=${PDS_INVITE_CODE_REQUIRED:-true}
15
+
- PDS_ROTATION_KEY=${PDS_ROTATION_KEY:?}
16
+
- PDS_JWK_MULTIBASE=${PDS_JWK_MULTIBASE:?}
17
+
- PDS_ADMIN_PASSWORD=${PDS_ADMIN_PASSWORD:?}
18
+
- PDS_CRAWLERS=${PDS_CRAWLERS:-https://bsky.network}
19
+
- PDS_DPOP_NONCE_SECRET=${PDS_DPOP_NONCE_SECRET}
50
20
51
-
# (optional) email config (see readme for details)
52
-
# auth uri should look like smtp[s]://user:pass@host[:port]
53
-
# sender should look like e@mail.com or Name <e@mail.com>
54
-
# without setting these, emails will be logged to stdout
55
-
# - PDS_SMTP_STARTTLS=false
56
-
# - PDS_SMTP_AUTH_URI=
57
-
# - PDS_SMTP_SENDER=
21
+
- PDS_SMTP_STARTTLS=${PDS_SMTP_STARTTLS:-false}
22
+
- PDS_SMTP_AUTH_URI=${PDS_SMTP_AUTH_URI}
23
+
- PDS_SMTP_SENDER=${PDS_SMTP_SENDER}
58
24
59
-
# (optional) S3 config (see readme for details)
60
-
# set these to back up your databases and/or blobs to S3(-compatible storge)
61
-
# - PDS_S3_BLOBS_ENABLED=false
62
-
# - PDS_S3_BACKUPS_ENABLED=false
63
-
# - PDS_S3_BACKUP_INTERVAL_S=3600
64
-
# - PDS_S3_ENDPOINT=
65
-
# - PDS_S3_REGION=
66
-
# - PDS_S3_BUCKET=
67
-
# - PDS_S3_ACCESS_KEY=
68
-
# - PDS_S3_SECRET_KEY=
69
-
# - PDS_S3_CDN_URL=
25
+
- PDS_S3_BLOBS_ENABLED=${PDS_S3_BLOBS_ENABLED:-false}
26
+
- PDS_S3_BACKUPS_ENABLED=${PDS_S3_BACKUPS_ENABLED:-false}
27
+
- PDS_S3_BACKUP_INTERVAL_S=${PDS_S3_BACKUP_INTERVAL_S:-3600}
28
+
- PDS_S3_ENDPOINT=${PDS_S3_ENDPOINT}
29
+
- PDS_S3_REGION=${PDS_S3_REGION}
30
+
- PDS_S3_BUCKET=${PDS_S3_BUCKET}
31
+
- PDS_S3_ACCESS_KEY=${PDS_S3_ACCESS_KEY}
32
+
- PDS_S3_SECRET_KEY=${PDS_S3_SECRET_KEY}
33
+
- PDS_S3_CDN_URL=${PDS_S3_CDN_URL}
70
34
restart: unless-stopped
71
35
72
36
caddy:
···
79
43
- caddy-data:/data
80
44
- caddy-config:/config
81
45
environment:
82
-
- PDS_HOSTNAME=
46
+
- PDS_HOSTNAME=${PDS_HOSTNAME:?}
83
47
restart: unless-stopped
84
48
85
49
volumes:
+73
-68
pegasus/lib/env.ml
+73
-68
pegasus/lib/env.ml
···
2
2
try Sys.getenv name
3
3
with Not_found -> failwith ("Missing environment variable " ^ name)
4
4
5
+
let getenv_opt name ~default =
6
+
match Sys.getenv_opt name with
7
+
| Some value when value <> "" ->
8
+
value
9
+
| _ ->
10
+
default
11
+
5
12
let log_level =
6
-
match Sys.getenv_opt "PDS_LOG_LEVEL" |> Option.map String.lowercase_ascii with
7
-
| Some "debug" ->
13
+
match getenv_opt "PDS_LOG_LEVEL" ~default:"info" with
14
+
| "debug" ->
8
15
`Debug
9
-
| Some "info" ->
10
-
`Info
11
-
| Some "warn" | Some "warning" ->
16
+
| "warn" | "warning" ->
12
17
`Warning
13
-
| Some "error" ->
18
+
| "error" ->
14
19
`Error
15
20
| _ ->
16
21
`Info
17
22
18
-
let data_dir = Option.value ~default:"./data" @@ Sys.getenv_opt "PDS_DATA_DIR"
23
+
let data_dir = getenv_opt "PDS_DATA_DIR" ~default:"./data"
19
24
20
25
let hostname = getenv "PDS_HOSTNAME"
21
26
22
27
let host_endpoint = "https://" ^ hostname
23
28
24
-
let did =
25
-
Option.value ~default:("did:web:" ^ hostname) @@ Sys.getenv_opt "PDS_DID"
29
+
let did = getenv_opt "PDS_DID" ~default:("did:web:" ^ hostname)
26
30
27
31
let invite_required = getenv "PDS_INVITE_CODE_REQUIRED" = "true"
28
32
···
36
40
let admin_password = getenv "PDS_ADMIN_PASSWORD"
37
41
38
42
let crawlers =
39
-
Sys.getenv_opt "PDS_CRAWLERS"
40
-
|> Option.value ~default:"https://bsky.network"
43
+
getenv_opt "PDS_CRAWLERS" ~default:"https://bsky.network"
41
44
|> String.split_on_char ','
42
45
|> List.map (fun u ->
43
46
match Uri.of_string @@ String.trim u with
···
47
50
Uri.make ~scheme:"https" ~host:u () )
48
51
49
52
let dpop_nonce_secret =
50
-
match Sys.getenv_opt "PDS_DPOP_NONCE_SECRET" with
51
-
| Some sec ->
53
+
match getenv_opt "PDS_DPOP_NONCE_SECRET" ~default:"" with
54
+
| "" ->
55
+
let secret = Mirage_crypto_rng_unix.getrandom 32 in
56
+
Dream.warning (fun log ->
57
+
log "PDS_DPOP_NONCE_SECRET not set; using PDS_DPOP_NONCE_SECRET=%s"
58
+
( Base64.(encode ~alphabet:uri_safe_alphabet ~pad:false) secret
59
+
|> Result.get_ok ) ) ;
60
+
Bytes.of_string secret
61
+
| sec ->
52
62
let secret =
53
63
Base64.(decode_exn ~alphabet:uri_safe_alphabet ~pad:false) sec
54
64
|> Bytes.of_string
55
65
in
56
66
if Bytes.length secret = 32 then secret
57
67
else failwith "PDS_DPOP_NONCE_SECRET must be 32 bytes in base64uri"
58
-
| None ->
59
-
let secret = Mirage_crypto_rng_unix.getrandom 32 in
60
-
Dream.warning (fun log ->
61
-
log "PDS_DPOP_NONCE_SECRET not set; using PDS_DPOP_NONCE_SECRET=%s"
62
-
( Base64.(encode ~alphabet:uri_safe_alphabet ~pad:false) secret
63
-
|> Result.get_ok ) ) ;
64
-
Bytes.of_string secret
65
68
66
69
let smtp_config, smtp_sender =
67
70
begin
68
71
let with_starttls =
69
-
Option.value ~default:"false" @@ Sys.getenv_opt "PDS_SMTP_STARTTLS"
70
-
= "true"
72
+
getenv_opt "PDS_SMTP_STARTTLS" ~default:"false" = "true"
71
73
in
72
74
match
73
-
( Option.map Uri.of_string (Sys.getenv_opt "PDS_SMTP_AUTH_URI")
74
-
, Sys.getenv_opt "PDS_SMTP_SENDER" )
75
+
( getenv_opt "PDS_SMTP_AUTH_URI" ~default:""
76
+
, getenv_opt "PDS_SMTP_SENDER" ~default:"" )
75
77
with
76
-
| Some uri, Some sender -> (
77
-
match
78
-
( Uri.scheme uri
79
-
, Uri.user uri
80
-
, Uri.password uri
81
-
, Uri.host uri
82
-
, Uri.port uri )
83
-
with
84
-
| Some scheme, Some username, Some password, Some hostname, port
85
-
when scheme = "smtp" || scheme = "smtps" -> (
86
-
match Emile.of_string sender with
87
-
| Ok _ ->
88
-
( Some
89
-
Letters.Config.(
90
-
create ~username ~password ~hostname ~with_starttls ()
91
-
|> set_port port )
92
-
, Some sender )
93
-
| Error _ ->
94
-
failwith
95
-
"PDS_SMTP_SENDER should be a valid mailbox, e.g. `e@mail.com` or \
96
-
`Name <e@mail.com>`" )
97
-
| _ ->
98
-
failwith
99
-
"PDS_SMTP_AUTH_URI must be a valid smtp:// or smtps:// URI with \
100
-
username, password, and hostname" )
101
-
| Some _, None ->
78
+
| "", "" ->
79
+
(None, None)
80
+
| _uri, "" ->
102
81
failwith
103
82
"PDS_SMTP_SENDER must be set alongside PDS_SMTP_AUTH_URI; it should \
104
83
look like `e@mail.com` or `Name <e@mail.com>`"
105
-
| None, Some _ ->
84
+
| "", _uri ->
106
85
failwith "PDS_SMTP_AUTH_URI must be set alongside PDS_SMTP_SENDER"
107
-
| None, None ->
108
-
(None, None)
86
+
| uri, sender -> (
87
+
let uri = Uri.of_string uri in
88
+
match
89
+
( Uri.scheme uri
90
+
, Uri.user uri
91
+
, Uri.password uri
92
+
, Uri.host uri
93
+
, Uri.port uri )
94
+
with
95
+
| Some scheme, Some username, Some password, Some hostname, port
96
+
when scheme = "smtp" || scheme = "smtps" -> (
97
+
match Emile.of_string sender with
98
+
| Ok _ ->
99
+
( Some
100
+
Letters.Config.(
101
+
create ~username ~password ~hostname ~with_starttls ()
102
+
|> set_port port )
103
+
, Some sender )
104
+
| Error _ ->
105
+
failwith
106
+
"PDS_SMTP_SENDER should be a valid mailbox, e.g. `e@mail.com` \
107
+
or `Name <e@mail.com>`" )
108
+
| _ ->
109
+
failwith
110
+
"PDS_SMTP_AUTH_URI must be a valid smtp:// or smtps:// URI with \
111
+
username, password, and hostname" )
109
112
end
110
113
111
114
type s3_config =
···
125
128
begin
126
129
let default_backup_interval = 3600.0 in
127
130
let blobs_enabled =
128
-
Sys.getenv_opt "PDS_S3_BLOBS_ENABLED" |> Option.map (( = ) "true")
131
+
getenv_opt "PDS_S3_BLOBS_ENABLED" ~default:"false" = "true"
129
132
in
130
133
let backups_enabled =
131
-
Sys.getenv_opt "PDS_S3_BACKUPS_ENABLED" |> Option.map (( = ) "true")
134
+
getenv_opt "PDS_S3_BACKUPS_ENABLED" ~default:"false" = "true"
132
135
in
133
136
let backup_interval_s =
134
-
Sys.getenv_opt "PDS_S3_BACKUP_INTERVAL_S"
135
-
|> Option.map float_of_string_opt
136
-
|> Option.join
137
+
getenv_opt "PDS_S3_BACKUP_INTERVAL_S"
138
+
~default:(string_of_float default_backup_interval)
139
+
|> float_of_string
137
140
in
138
141
let endpoint = Sys.getenv_opt "PDS_S3_ENDPOINT" in
139
142
match (blobs_enabled, backups_enabled) with
140
-
| Some true, _ | _, Some true -> (
143
+
| true, _ | _, true -> (
141
144
match
142
145
( Sys.getenv_opt "PDS_S3_REGION"
143
146
, Sys.getenv_opt "PDS_S3_BUCKET"
144
147
, Sys.getenv_opt "PDS_S3_ACCESS_KEY"
145
-
, Sys.getenv_opt "PDS_S3_SECRET_KEY" )
148
+
, Sys.getenv_opt "PDS_S3_SECRET_KEY"
149
+
, Sys.getenv_opt "PDS_S3_CDN_URL" )
146
150
with
147
-
| Some region, Some bucket, Some access_key, Some secret_key ->
151
+
| Some region, Some bucket, Some access_key, Some secret_key, cdn_url ->
148
152
let region_obj =
149
153
match endpoint with
150
154
| Some host ->
···
155
159
let endpoint_obj =
156
160
Aws_s3.Region.endpoint ~inet:`V4 ~scheme:`Https region_obj
157
161
in
162
+
let credentials_obj =
163
+
Aws_s3.Credentials.make ~access_key ~secret_key ()
164
+
in
158
165
Some
159
-
{ blobs_enabled= Option.value ~default:false blobs_enabled
160
-
; backups_enabled= Option.value ~default:false backups_enabled
161
-
; backup_interval_s=
162
-
Option.value ~default:default_backup_interval backup_interval_s
166
+
{ blobs_enabled
167
+
; backups_enabled
168
+
; backup_interval_s
163
169
; endpoint
164
170
; region
165
171
; bucket
166
172
; access_key
167
173
; secret_key
168
-
; cdn_url= Sys.getenv_opt "PDS_S3_CDN_URL"
169
-
; credentials_obj=
170
-
Aws_s3.Credentials.make ~access_key ~secret_key ()
174
+
; cdn_url
175
+
; credentials_obj
171
176
; endpoint_obj }
172
177
| _ ->
173
178
failwith