Our Personal Data Server from scratch! tranquil.farm
oauth atproto pds rust postgresql objectstorage fun

feat: nix module #16

merged opened by lewis.moe targeting main from feat/nix-module

Added a module and a test file to test it out! Tested locally on qemu

Labels

None yet.

assignee

None yet.

Participants 2
Referenced by
AT URI
at://did:plc:3fwecdnvtcscjnrx2p4n7alz/sh.tangled.repo.pull/3me4wjqzoiv22
+22 -28
Interdiff #4 โ†’ #5
default.nix

This file has not been changed.

docs/install-debian.md

This file has not been changed.

flake.nix

This file has not been changed.

+9 -15
module.nix
··· 149 }; 150 151 SENDMAIL_PATH = mkOption { 152 - type = types.nullOr types.path; 153 - default = null; 154 description = "Path to the sendmail executable to use for sending emails."; 155 }; 156 ··· 161 }; 162 163 SIGNAL_CLI_PATH = mkOption { 164 - type = types.nullOr types.path; 165 - default = null; 166 description = "Path to the signal-cli executable to use for sending Signal notifications."; 167 }; 168 ··· 309 }) 310 311 { 312 - services.tranquil-pds.settings = { 313 - SENDMAIL_PATH = lib.mkDefault ( 314 - if cfg.settings.MAIL_FROM_ADDRESS != null then (lib.getExe pkgs.system-sendmail) else null 315 - ); 316 - 317 - SIGNAL_CLI_PATH = lib.mkDefault ( 318 - if cfg.settings.SIGNAL_SENDER_NUMBER != null then (lib.getExe pkgs.signal-cli) else null 319 - ); 320 - }; 321 - 322 users.users.${cfg.user} = { 323 isSystemUser = true; 324 inherit (cfg) group; ··· 359 360 EnvironmentFile = cfg.environmentFiles; 361 Environment = lib.mapAttrsToList (k: v: "${k}=${if builtins.isInt v then toString v else v}") ( 362 - lib.filterAttrs (_: v: v != null) cfg.settings 363 ); 364 365 NoNewPrivileges = true;
··· 149 }; 150 151 SENDMAIL_PATH = mkOption { 152 + type = types.path; 153 + default = lib.getExe pkgs.system-sendmail; 154 description = "Path to the sendmail executable to use for sending emails."; 155 }; 156 ··· 161 }; 162 163 SIGNAL_CLI_PATH = mkOption { 164 + type = types.path; 165 + default = lib.getExe pkgs.signal-cli; 166 description = "Path to the signal-cli executable to use for sending Signal notifications."; 167 }; 168 ··· 309 }) 310 311 { 312 users.users.${cfg.user} = { 313 isSystemUser = true; 314 inherit (cfg) group; ··· 349 350 EnvironmentFile = cfg.environmentFiles; 351 Environment = lib.mapAttrsToList (k: v: "${k}=${if builtins.isInt v then toString v else v}") ( 352 + lib.filterAttrs (k: v: 353 + if k == "SENDMAIL_PATH" then cfg.settings.MAIL_FROM_ADDRESS != null 354 + else if k == "SIGNAL_CLI_PATH" then cfg.settings.SIGNAL_SENDER_NUMBER != null 355 + else v != null 356 + ) cfg.settings 357 ); 358 359 NoNewPrivileges = true;
+13 -13
test.nix
··· 23 }; 24 25 settings = { 26 - PDS_HOSTNAME = "test.local"; 27 SERVER_HOST = "0.0.0.0"; 28 29 DISABLE_RATE_LIMITING = 1; ··· 46 server.wait_for_open_port(80) 47 48 def xrpc(method, endpoint, *, headers=None, data=None, raw_body=None, via="nginx"): 49 - host_header = "-H 'Host: test.local'" if via == "nginx" else "" 50 base = "http://localhost" if via == "nginx" else "http://localhost:3000" 51 url = f"{base}/xrpc/{endpoint}" 52 ··· 66 return json.loads(xrpc(method, endpoint, **kwargs)) 67 68 def xrpc_status(endpoint, *, headers=None, via="nginx"): 69 - host_header = "-H 'Host: test.local'" if via == "nginx" else "" 70 base = "http://localhost" if via == "nginx" else "http://localhost:3000" 71 url = f"{base}/xrpc/{endpoint}" 72 ··· 77 78 return server.succeed(" ".join(parts)).strip() 79 80 - def http_status(path, *, host="test.local", via="nginx"): 81 base = "http://localhost" if via == "nginx" else "http://localhost:3000" 82 return server.succeed( 83 f"curl -s -o /dev/null -w '%{{http_code}}' -H 'Host: {host}' '{base}{path}'" 84 ).strip() 85 86 - def http_get(path, *, host="test.local"): 87 return server.succeed( 88 f"curl -sf -H 'Host: {host}' 'http://localhost{path}'" 89 ) 90 91 - def http_header(path, header, *, host="test.local"): 92 return server.succeed( 93 f"curl -sI -H 'Host: {host}' 'http://localhost{path}'" 94 f" | grep -i '^{header}:'" ··· 120 assert desc.get("inviteCodeRequired") == False 121 122 with subtest("nginx serves frontend"): 123 - result = server.succeed("curl -sf -H 'Host: test.local' http://localhost/") 124 assert "<html" in result.lower() or "<!" in result 125 126 with subtest("well-known proxied"): ··· 144 assert code != "502" and code != "504", f"oauth proxy broken: {code}" 145 146 with subtest("subdomain routing works"): 147 - code = http_status("/xrpc/_health", host="alice.test.local") 148 assert code == "200", f"subdomain routing failed: {code}" 149 150 with subtest("client-metadata.json served with host substitution"): 151 meta_raw = http_get("/oauth/client-metadata.json") 152 meta = json.loads(meta_raw) 153 assert "client_id" in meta, f"no client_id in client-metadata: {meta}" 154 - assert "test.local" in meta_raw, "host substitution did not apply" 155 156 with subtest("static assets location exists"): 157 code = http_status("/assets/nonexistent.js") ··· 169 170 with subtest("create account"): 171 account = xrpc_json("POST", "com.atproto.server.createAccount", data={ 172 - "handle": "alice.test.local", 173 "password": "NixOS-Test-Pass-99!", 174 - "email": "alice@test.local", 175 "didType": "web", 176 }) 177 assert "accessJwt" in account, f"no accessJwt: {account}" ··· 191 with subtest("get session"): 192 session = xrpc_json("GET", "com.atproto.server.getSession", headers=auth) 193 assert session["did"] == did 194 - assert session["handle"] == "alice.test.local" 195 196 with subtest("create record"): 197 created = xrpc_json("POST", "com.atproto.repo.createRecord", headers=auth, data={ ··· 232 233 with subtest("export repo as car"): 234 server.succeed( 235 - f"curl -sf -H 'Host: test.local' " 236 f"-o /tmp/repo.car " 237 f"'http://localhost/xrpc/com.atproto.sync.getRepo?did={did}'" 238 )
··· 23 }; 24 25 settings = { 26 + PDS_HOSTNAME = "pds.test"; 27 SERVER_HOST = "0.0.0.0"; 28 29 DISABLE_RATE_LIMITING = 1; ··· 46 server.wait_for_open_port(80) 47 48 def xrpc(method, endpoint, *, headers=None, data=None, raw_body=None, via="nginx"): 49 + host_header = "-H 'Host: pds.test'" if via == "nginx" else "" 50 base = "http://localhost" if via == "nginx" else "http://localhost:3000" 51 url = f"{base}/xrpc/{endpoint}" 52 ··· 66 return json.loads(xrpc(method, endpoint, **kwargs)) 67 68 def xrpc_status(endpoint, *, headers=None, via="nginx"): 69 + host_header = "-H 'Host: pds.test'" if via == "nginx" else "" 70 base = "http://localhost" if via == "nginx" else "http://localhost:3000" 71 url = f"{base}/xrpc/{endpoint}" 72 ··· 77 78 return server.succeed(" ".join(parts)).strip() 79 80 + def http_status(path, *, host="pds.test", via="nginx"): 81 base = "http://localhost" if via == "nginx" else "http://localhost:3000" 82 return server.succeed( 83 f"curl -s -o /dev/null -w '%{{http_code}}' -H 'Host: {host}' '{base}{path}'" 84 ).strip() 85 86 + def http_get(path, *, host="pds.test"): 87 return server.succeed( 88 f"curl -sf -H 'Host: {host}' 'http://localhost{path}'" 89 ) 90 91 + def http_header(path, header, *, host="pds.test"): 92 return server.succeed( 93 f"curl -sI -H 'Host: {host}' 'http://localhost{path}'" 94 f" | grep -i '^{header}:'" ··· 120 assert desc.get("inviteCodeRequired") == False 121 122 with subtest("nginx serves frontend"): 123 + result = server.succeed("curl -sf -H 'Host: pds.test' http://localhost/") 124 assert "<html" in result.lower() or "<!" in result 125 126 with subtest("well-known proxied"): ··· 144 assert code != "502" and code != "504", f"oauth proxy broken: {code}" 145 146 with subtest("subdomain routing works"): 147 + code = http_status("/xrpc/_health", host="alice.pds.test") 148 assert code == "200", f"subdomain routing failed: {code}" 149 150 with subtest("client-metadata.json served with host substitution"): 151 meta_raw = http_get("/oauth/client-metadata.json") 152 meta = json.loads(meta_raw) 153 assert "client_id" in meta, f"no client_id in client-metadata: {meta}" 154 + assert "pds.test" in meta_raw, "host substitution did not apply" 155 156 with subtest("static assets location exists"): 157 code = http_status("/assets/nonexistent.js") ··· 169 170 with subtest("create account"): 171 account = xrpc_json("POST", "com.atproto.server.createAccount", data={ 172 + "handle": "alice", 173 "password": "NixOS-Test-Pass-99!", 174 + "email": "alice@pds.test", 175 "didType": "web", 176 }) 177 assert "accessJwt" in account, f"no accessJwt: {account}" ··· 191 with subtest("get session"): 192 session = xrpc_json("GET", "com.atproto.server.getSession", headers=auth) 193 assert session["did"] == did 194 + assert session["handle"] == "alice.pds.test", f"unexpected handle: {session['handle']}" 195 196 with subtest("create record"): 197 created = xrpc_json("POST", "com.atproto.repo.createRecord", headers=auth, data={ ··· 232 233 with subtest("export repo as car"): 234 server.succeed( 235 + f"curl -sf -H 'Host: pds.test' " 236 f"-o /tmp/repo.car " 237 f"'http://localhost/xrpc/com.atproto.sync.getRepo?did={did}'" 238 )

History

8 rounds 13 comments
sign up or login to add to the discussion
6 commits
expand
feat: nix module
fix: better defaults, add in pg & nginx
feat: actual good config from Isabel
fix: set env vars of paths for sendmail & signal from nix pkg not manual
refactor(nix): toml conf
fix: minor format cleanup and description fixing in the module
expand 0 comments
pull request successfully merged
6 commits
expand
feat: nix module
fix: better defaults, add in pg & nginx
feat: actual good config from Isabel
fix: set env vars of paths for sendmail & signal from nix pkg not manual
refactor(nix): toml conf
fix: minor format cleanup and description fixing in the module
expand 0 comments
lewis.moe submitted #5
4 commits
expand
feat: nix module
fix: better defaults, add in pg & nginx
feat: actual good config from Isabel
fix: set env vars of paths for sendmail & signal from nix pkg not manual
expand 0 comments
3 commits
expand
feat: nix module
fix: better defaults, add in pg & nginx
feat: actual good config from Isabel
expand 0 comments
4 commits
expand
feat: nix module
fix: better defaults, add in pg & nginx
fix: bulk type safety improvements, added a couple of tests
feat: actual good config from Isabel
expand 0 comments
2 commits
expand
feat: nix module
fix: better defaults, add in pg & nginx
expand 5 comments

Generally much better! I'll go through with a more fine-toothed comb at some point, but one immediate comment is that you should probably be using lib.mkEnableOption rather than lib.mkOption for your enable options - it's more idiomatic.

I'd also be interested in seeing your default paths for sendmail and signal moved up (and the options can be made no longer nullOr in that case too) - remember that in Nix nothing is evaluated unless you actually look at it, so as long as you check that email/signal are enabled when you place those options into the environment file (e.g. with mkIf) then this still needn't cause a dependency when it's not desirable...

https://search.nixos.org/packages?channel=25.11&show=system-sendmail&query=sendmail

^ You should probably also consider using system-sendmail which looks in a few places rather than your own best-guess sendmail path

The starred alias in nginx should really be tied to availableUserDomains rather than the hostname

I'm not a huge fan of some of the nginx stuff you're doing - recommendedProxySettings, etc. are good settings but you're leaking config outside the module which might change stuff for other modules. It might be better to enable recommendedProxySettings per-location and then setup whatever settings for the other categories you feel work best for tranquil in a lower scope...

...on a similar note, openFirewall tends to default to false to avoid unknowingly opening ports in firewalls that you didn't mean to

openFirewall tends to default to false

(I think there's one place in nixpkgs this isn't true - and it's with SSH to avoid making stuff unreachable unexpectedly)

7 commits
expand
feat: nix module
feat: fun handle resolution during migration
feat: initial in-house cache distribution
feat: cache locks less
fix: smaller docker img
fix: removed some bs
fix: better defaults, add in pg & nginx
expand 1 comment

Sorry, just updating to main

lewis.moe submitted #0
1 commit
expand
feat: nix module
expand 7 comments

Is there any reason the package doesn't seem to default to packages.tranquil-pds from this flake? I think it might be kinda nice if it did... (though I guess it would mean you have to have a reference to the flake in your module - that shouldn't really be much of a problem though)

What's the default for settings.storage.blobPath - is it actually null? what happens if I leave the blob backend as filesystem and then this as null? Could we default it in terms of dataDir (as I suspect it must really be) so that it's easier to introspect or could we put what this'll be in the description of the option

Why is backup.backend set in the test file - given it has the same value as the default and backups aren't enabled by default (or are they? they're set to null?) ... is this a mistake or something else? blobBackend is also set to its default but this is a little more understandable incase the default were to change/etc.

I also wonder if backup.enabled should be called backup.enable and should be a mkEnableOption rather than what you are doing here...... it seems quite unidiomatic

You're using optional types in a bunch of places. I'm going to assume that they are being set as overrides to some other config that is secretly elsewhere (e.g. plc.directoryUrl)... it would be much more idiomatic to have these required and have the true defaults in nix

Can you default sendmailPath (and signal CLI path) to whatever it should be from nixpkgs? I think that'd be nice :) - you need to make sure these options don't get evaluated unless mail or signal is enabled respectively

Thanks for all the comments! I am no nix connoisseur and i'll read through carefully and try to fix