parakeet.at
Parakeet is a Rust-based Bluesky AppServer aiming to implement most of the functionality required to support the Bluesky client
appview
atproto
bluesky
rust
appserver
feat: nix
mia.omg.lol
+24
-6
Cargo.lock
+24
-6
Cargo.lock
···
197
197
source = "registry+https://github.com/rust-lang/crates.io-index"
198
198
checksum = "0f9dd2e03ee80ca2822dd6ea431163d2ef259f2066a4d6ccaca6d9dcb386aa43"
199
199
dependencies = [
200
-
"bindgen",
200
+
"bindgen 0.69.5",
201
201
"cc",
202
202
"cmake",
203
203
"dunce",
···
370
370
"shlex",
371
371
"syn",
372
372
"which",
373
+
]
374
+
375
+
[[package]]
376
+
name = "bindgen"
377
+
version = "0.72.1"
378
+
source = "registry+https://github.com/rust-lang/crates.io-index"
379
+
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
380
+
dependencies = [
381
+
"bitflags 2.8.0",
382
+
"cexpr",
383
+
"clang-sys",
384
+
"itertools 0.12.1",
385
+
"proc-macro2",
386
+
"quote",
387
+
"regex",
388
+
"rustc-hash 2.1.1",
389
+
"shlex",
390
+
"syn",
373
391
]
374
392
375
393
[[package]]
···
2129
2147
2130
2148
[[package]]
2131
2149
name = "librocksdb-sys"
2132
-
version = "0.17.1+9.9.3"
2150
+
version = "0.17.3+10.4.2"
2133
2151
source = "registry+https://github.com/rust-lang/crates.io-index"
2134
-
checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f"
2152
+
checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9"
2135
2153
dependencies = [
2136
-
"bindgen",
2154
+
"bindgen 0.72.1",
2137
2155
"bzip2-sys",
2138
2156
"cc",
2139
2157
"libc",
···
3459
3477
3460
3478
[[package]]
3461
3479
name = "rocksdb"
3462
-
version = "0.23.0"
3480
+
version = "0.24.0"
3463
3481
source = "registry+https://github.com/rust-lang/crates.io-index"
3464
-
checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43"
3482
+
checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f"
3465
3483
dependencies = [
3466
3484
"libc",
3467
3485
"librocksdb-sys",
+7
Cargo.toml
+7
Cargo.toml
+6
crates/consumer/src/config.rs
+6
crates/consumer/src/config.rs
···
35
35
pub struct ConfigInstruments {
36
36
#[serde(default)]
37
37
pub log_json: bool,
38
+
#[serde(default = "default_metrics_port")]
39
+
pub metrics_port: u16,
38
40
}
39
41
40
42
#[derive(Debug, Deserialize)]
···
73
75
#[serde(default = "default_download_buffer")]
74
76
pub download_buffer: usize,
75
77
pub download_tmp_dir: String,
78
+
}
79
+
80
+
fn default_metrics_port() -> u16 {
81
+
9000
76
82
}
77
83
78
84
fn default_backfill_workers() -> u8 {
+7
-2
crates/consumer/src/main.rs
+7
-2
crates/consumer/src/main.rs
···
18
18
19
19
#[tokio::main]
20
20
async fn main() -> eyre::Result<()> {
21
-
PrometheusBuilder::new().install()?;
22
-
23
21
let cli = cmd::parse();
24
22
let conf = config::load_config()?;
23
+
24
+
PrometheusBuilder::new()
25
+
.with_http_listener(std::net::SocketAddr::new(
26
+
[0, 0, 0, 0].into(),
27
+
conf.instruments.metrics_port,
28
+
))
29
+
.install()?;
25
30
26
31
instrumentation::init_instruments(&conf.instruments);
27
32
let user_agent = build_ua(&conf.ua_contact);
+2
-2
crates/parakeet-index/Cargo.toml
+2
-2
crates/parakeet-index/Cargo.toml
···
19
19
opentelemetry = { version = "0.31.0", optional = true }
20
20
opentelemetry-otlp = { version = "0.31.0", features = ["reqwest-rustls"], optional = true }
21
21
opentelemetry_sdk = { version = "0.31.0", optional = true }
22
-
rocksdb = { version = "0.23", default-features = false, features = ["lz4", "bindgen-runtime"], optional = true }
22
+
rocksdb = { version = "0.24", default-features = false, features = ["lz4", "bindgen-runtime"], optional = true }
23
23
serde = { version = "1.0.217", features = ["derive"], optional = true }
24
24
tokio = { version = "1.42.0", features = ["full"], optional = true }
25
25
tonic-health = { version = "0.13.0", optional = true }
···
47
47
"dep:tracing",
48
48
"dep:tracing-subscriber",
49
49
"dep:tracing-opentelemetry"
50
-
]
50
+
]
+4
-1
crates/parakeet-index/build.rs
+4
-1
crates/parakeet-index/build.rs
···
1
1
fn main() -> Result<(), Box<dyn std::error::Error>> {
2
-
tonic_build::configure().compile_protos(&["proto/parakeet.proto"], &[""])?;
2
+
tonic_build::configure().compile_protos(
3
+
&[std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("proto/parakeet.proto")],
4
+
&[std::path::Path::new(env!("CARGO_MANIFEST_DIR"))],
5
+
)?;
3
6
4
7
Ok(())
5
8
}
+98
flake.lock
+98
flake.lock
···
1
+
{
2
+
"nodes": {
3
+
"crane": {
4
+
"locked": {
5
+
"lastModified": 1768319649,
6
+
"narHash": "sha256-VFkNyxHxkqGp8gf8kfFMW1j6XeBy609kv6TE9uF/0Js=",
7
+
"owner": "ipetkov",
8
+
"repo": "crane",
9
+
"rev": "4b6527687cfd20da3c2ef8287e01b74c2d6c705b",
10
+
"type": "github"
11
+
},
12
+
"original": {
13
+
"owner": "ipetkov",
14
+
"repo": "crane",
15
+
"type": "github"
16
+
}
17
+
},
18
+
"flake-utils": {
19
+
"inputs": {
20
+
"systems": "systems"
21
+
},
22
+
"locked": {
23
+
"lastModified": 1731533236,
24
+
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
25
+
"owner": "numtide",
26
+
"repo": "flake-utils",
27
+
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
28
+
"type": "github"
29
+
},
30
+
"original": {
31
+
"owner": "numtide",
32
+
"repo": "flake-utils",
33
+
"type": "github"
34
+
}
35
+
},
36
+
"nixpkgs": {
37
+
"locked": {
38
+
"lastModified": 1768395095,
39
+
"narHash": "sha256-ZhuYJbwbZT32QA95tSkXd9zXHcdZj90EzHpEXBMabaw=",
40
+
"owner": "nixos",
41
+
"repo": "nixpkgs",
42
+
"rev": "13868c071cc73a5e9f610c47d7bb08e5da64fdd5",
43
+
"type": "github"
44
+
},
45
+
"original": {
46
+
"owner": "nixos",
47
+
"ref": "nixpkgs-unstable",
48
+
"repo": "nixpkgs",
49
+
"type": "github"
50
+
}
51
+
},
52
+
"root": {
53
+
"inputs": {
54
+
"crane": "crane",
55
+
"flake-utils": "flake-utils",
56
+
"nixpkgs": "nixpkgs",
57
+
"rust-overlay": "rust-overlay"
58
+
}
59
+
},
60
+
"rust-overlay": {
61
+
"inputs": {
62
+
"nixpkgs": [
63
+
"nixpkgs"
64
+
]
65
+
},
66
+
"locked": {
67
+
"lastModified": 1769222645,
68
+
"narHash": "sha256-gu6oZ86zLudBZMq8LL1qdtYt/S69GV5keQVXdvBrVSU=",
69
+
"owner": "oxalica",
70
+
"repo": "rust-overlay",
71
+
"rev": "22da29e7f3d8cff75009cbbcf992c7cb66920cfd",
72
+
"type": "github"
73
+
},
74
+
"original": {
75
+
"owner": "oxalica",
76
+
"repo": "rust-overlay",
77
+
"type": "github"
78
+
}
79
+
},
80
+
"systems": {
81
+
"locked": {
82
+
"lastModified": 1681028828,
83
+
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
84
+
"owner": "nix-systems",
85
+
"repo": "default",
86
+
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
87
+
"type": "github"
88
+
},
89
+
"original": {
90
+
"owner": "nix-systems",
91
+
"repo": "default",
92
+
"type": "github"
93
+
}
94
+
}
95
+
},
96
+
"root": "root",
97
+
"version": 7
98
+
}
+145
flake.nix
+145
flake.nix
···
1
+
{
2
+
description = "Parakeet - a Rust-based Bluesky AppView for Bluesky client";
3
+
4
+
inputs = {
5
+
nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
6
+
crane.url = "github:ipetkov/crane";
7
+
flake-utils.url = "github:numtide/flake-utils";
8
+
rust-overlay = {
9
+
url = "github:oxalica/rust-overlay";
10
+
inputs.nixpkgs.follows = "nixpkgs";
11
+
};
12
+
};
13
+
14
+
outputs =
15
+
{
16
+
self,
17
+
nixpkgs,
18
+
crane,
19
+
flake-utils,
20
+
rust-overlay,
21
+
...
22
+
}:
23
+
flake-utils.lib.eachDefaultSystem (
24
+
system:
25
+
let
26
+
pkgs = import nixpkgs {
27
+
inherit system;
28
+
overlays = [ (import rust-overlay) ];
29
+
};
30
+
31
+
inherit (pkgs) lib;
32
+
inherit (lib.fileset) toSource unions union;
33
+
34
+
craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default);
35
+
src = toSource {
36
+
root = ./.;
37
+
fileset = unions [ (craneLib.fileset.commonCargoSources ./.) ];
38
+
};
39
+
40
+
commonArgs = {
41
+
inherit src;
42
+
strictDeps = true;
43
+
44
+
nativeBuildInputs = with pkgs; [ pkg-config ];
45
+
buildInputs = with pkgs; [
46
+
clang
47
+
libclang
48
+
openssl
49
+
protobuf
50
+
];
51
+
52
+
CLANG_PATH = "${pkgs.clang}/bin/clang";
53
+
LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
54
+
PROTOC = "${pkgs.protobuf}/bin/protoc";
55
+
PROTOC_INCLUDE = "${pkgs.protobuf}/include";
56
+
};
57
+
58
+
cargoArtifacts = craneLib.buildDepsOnly commonArgs;
59
+
60
+
individualCrateArgs = commonArgs // {
61
+
inherit cargoArtifacts;
62
+
inherit (craneLib.crateNameFromCargoToml { inherit src; }) version;
63
+
doCheck = false;
64
+
};
65
+
66
+
fileSetForCrate =
67
+
crate: add:
68
+
toSource {
69
+
root = ./.;
70
+
fileset = union (unions [
71
+
./Cargo.lock
72
+
./Cargo.toml
73
+
(craneLib.fileset.commonCargoSources ./crates/lexica)
74
+
(craneLib.fileset.commonCargoSources ./crates/parakeet-db)
75
+
(craneLib.fileset.commonCargoSources ./crates/parakeet-index)
76
+
./crates/parakeet-index/proto
77
+
(craneLib.fileset.commonCargoSources crate)
78
+
]) add;
79
+
};
80
+
81
+
parakeet-appview = craneLib.buildPackage (
82
+
individualCrateArgs
83
+
// {
84
+
pname = "parakeet";
85
+
cargoExtraArgs = "-p parakeet";
86
+
src = fileSetForCrate ./crates/parakeet (unions [
87
+
./migrations
88
+
(craneLib.fileset.commonCargoSources ./crates/dataloader-rs)
89
+
(craneLib.fileset.commonCargoSources ./crates/did-resolver)
90
+
(craneLib.fileset.commonCargoSources ./crates/parakeet)
91
+
./crates/parakeet/src/sql
92
+
]);
93
+
}
94
+
);
95
+
96
+
parakeet-consumer = craneLib.buildPackage (
97
+
individualCrateArgs
98
+
// {
99
+
pname = "consumer";
100
+
cargoExtraArgs = "-p consumer";
101
+
src = fileSetForCrate ./crates/consumer (unions [
102
+
./crates/did-resolver
103
+
./crates/consumer/src/db/sql
104
+
]);
105
+
}
106
+
);
107
+
108
+
parakeet-index = craneLib.buildPackage (
109
+
individualCrateArgs
110
+
// {
111
+
pname = "parakeet-index";
112
+
cargoExtraArgs = "-p parakeet-index --features server";
113
+
src = fileSetForCrate ./crates/parakeet-index (unions [ ]);
114
+
}
115
+
);
116
+
in
117
+
{
118
+
packages = {
119
+
default = parakeet-appview;
120
+
inherit parakeet-appview parakeet-consumer parakeet-index;
121
+
};
122
+
123
+
apps = {
124
+
parakeet-appview = flake-utils.lib.mkApp { drv = parakeet-appview; };
125
+
parakeet-consumer = flake-utils.lib.mkApp { drv = parakeet-consumer; };
126
+
parakeet-index = flake-utils.lib.mkApp { drv = parakeet-index; };
127
+
};
128
+
129
+
devShells.default = craneLib.devShell {
130
+
packages = with pkgs; [
131
+
diesel-cli
132
+
protobuf
133
+
];
134
+
};
135
+
}
136
+
)
137
+
// flake-utils.lib.eachDefaultSystemPassThrough (system: {
138
+
nixosModules = {
139
+
default = import ./nix/services.nix {
140
+
inherit system;
141
+
parakeetSelf = self;
142
+
};
143
+
};
144
+
});
145
+
}
+187
nix/services.nix
+187
nix/services.nix
···
1
+
{ parakeetSelf, system, ... }:
2
+
{
3
+
config,
4
+
lib,
5
+
pkgs,
6
+
...
7
+
}:
8
+
9
+
let
10
+
inherit (lib)
11
+
getExe
12
+
mkEnableOption
13
+
mkIf
14
+
mkOption
15
+
types
16
+
;
17
+
18
+
thisSystemPackages = parakeetSelf.packages.${system};
19
+
20
+
cfg = config.services.parakeet;
21
+
anyEnabled = cfg.appview.enable || cfg.consumer.enable || cfg.index.enable;
22
+
23
+
settingsFormat = pkgs.formats.toml { };
24
+
25
+
toml = {
26
+
appview = settingsFormat.generate "Config-appview.toml" cfg.appview.settings;
27
+
consumer = settingsFormat.generate "Config-consumer.toml" cfg.consumer.settings;
28
+
index = settingsFormat.generate "Config-index.toml" cfg.index.settings;
29
+
};
30
+
31
+
buildService =
32
+
sub:
33
+
let
34
+
fullName = "parakeet-${sub}";
35
+
thisCfg = cfg.${sub};
36
+
thisToml = toml.${sub};
37
+
in
38
+
{
39
+
description = "Parakeet ${sub}";
40
+
41
+
after = [ "network-online.target" ];
42
+
wants = [ "network-online.target" ];
43
+
wantedBy = [ "multi-user.target" ];
44
+
45
+
preStart = "cp -f ${thisToml} $STATE_DIRECTORY/Config.toml";
46
+
47
+
serviceConfig = {
48
+
User = "parakeet";
49
+
Group = "parakeet";
50
+
Restart = "always";
51
+
ExecStart = "${getExe thisCfg.package}";
52
+
53
+
RuntimeDirectory = fullName;
54
+
StateDirectory = fullName;
55
+
WorkingDirectory = "/var/lib/${fullName}";
56
+
57
+
Environment = [ "RUST_LOG=${thisCfg.settings.RUST_LOG}" ];
58
+
EnvironmentFile = mkIf (thisCfg.environmentFile != null) thisCfg.environmentFile;
59
+
};
60
+
};
61
+
in
62
+
{
63
+
options.services.parakeet = {
64
+
appview = {
65
+
enable = mkEnableOption "Parakeet Appview";
66
+
67
+
package = mkOption {
68
+
type = types.package;
69
+
default = thisSystemPackages.parakeet-appview;
70
+
description = "The parakeet-appview package to use";
71
+
};
72
+
73
+
environmentFile = mkOption {
74
+
type = types.nullOr types.path;
75
+
default = null;
76
+
description = "The environment file to use for parakeet-appview";
77
+
};
78
+
79
+
settings = mkOption {
80
+
type = types.submodule {
81
+
freeformType = settingsFormat.type;
82
+
83
+
options = {
84
+
RUST_LOG = mkOption {
85
+
type = types.str;
86
+
default = "info";
87
+
description = "Logging level and filters to use. Uses tracing-subscriber EnvFilter";
88
+
};
89
+
};
90
+
};
91
+
};
92
+
};
93
+
94
+
consumer = {
95
+
enable = mkEnableOption "Parakeet Consumer";
96
+
97
+
package = mkOption {
98
+
type = types.package;
99
+
default = thisSystemPackages.parakeet-consumer;
100
+
description = "The parakeet-consumer package to use";
101
+
};
102
+
103
+
environmentFile = mkOption {
104
+
type = lib.types.nullOr lib.types.path;
105
+
default = null;
106
+
description = "The environment file to use for parakeet-consumer";
107
+
};
108
+
109
+
settings = mkOption {
110
+
type = types.submodule {
111
+
freeformType = settingsFormat.type;
112
+
113
+
options = {
114
+
RUST_LOG = mkOption {
115
+
type = types.str;
116
+
default = "info";
117
+
description = "Logging level and filters to use. Uses tracing-subscriber EnvFilter";
118
+
};
119
+
120
+
modes = mkOption {
121
+
type = types.listOf types.str;
122
+
default = [ "indexer" ];
123
+
description = "Modes to run the consumer in. Accepts one or more of backfill, indexer, labels.";
124
+
};
125
+
};
126
+
};
127
+
};
128
+
};
129
+
130
+
index = {
131
+
enable = mkEnableOption "Parakeet Index";
132
+
133
+
package = mkOption {
134
+
type = types.package;
135
+
default = thisSystemPackages.parakeet-index;
136
+
description = "The parakeet-index package to use";
137
+
};
138
+
139
+
environmentFile = mkOption {
140
+
type = lib.types.nullOr lib.types.path;
141
+
default = null;
142
+
description = "The environment file to use for parakeet-index";
143
+
};
144
+
145
+
settings = mkOption {
146
+
type = types.submodule {
147
+
freeformType = settingsFormat.type;
148
+
149
+
options = {
150
+
RUST_LOG = mkOption {
151
+
type = types.str;
152
+
default = "info";
153
+
description = "Logging level and filters to use. Uses tracing-subscriber EnvFilter";
154
+
};
155
+
};
156
+
};
157
+
};
158
+
};
159
+
};
160
+
161
+
config = {
162
+
environment.systemPackages = mkIf cfg.consumer.enable [
163
+
thisSystemPackages.parakeet-consumer
164
+
];
165
+
166
+
systemd.services.parakeet-appview = mkIf cfg.appview.enable (buildService "appview");
167
+
168
+
systemd.services.parakeet-consumer = mkIf cfg.consumer.enable (
169
+
lib.recursiveUpdate (buildService "consumer") {
170
+
serviceConfig.ExecStart = "${getExe thisSystemPackages.parakeet-consumer} ${
171
+
lib.strings.join " " (lib.map (x: "--${x}") cfg.consumer.settings.modes)
172
+
}";
173
+
}
174
+
);
175
+
176
+
systemd.services.parakeet-index = mkIf cfg.index.enable (buildService "index");
177
+
178
+
users = mkIf anyEnabled {
179
+
users.parakeet = {
180
+
group = "parakeet";
181
+
isSystemUser = true;
182
+
};
183
+
184
+
groups.parakeet = { };
185
+
};
186
+
};
187
+
}