···11-# clippr-be
22-the reference appview for clippr, written in Dart and using the [atproto.dart](https://github.com/myConsciousness/atproto.dart) library
11+# clippr-ts
22+typescript implementation of clippr appview using ~~bun~~ deno and hono
33+44+## run and develop
55+```sh
66+deno install
77+deno run dev
88+```
99+1010+open http://localhost:9090 and enjoy
1111+1212+## current status
1313+right now we're not running on bun because there are
1414+[some issues with the jetstream library](https://github.com/oven-sh/bun/issues/18807), which haven't been fixed yet.
31544-## set up and run
55-```bash
66-cp config.example.yaml config.yaml
77-vi config.yaml # modify settings here
88-chmod +x tools/build_and_run.sh
99-./tools/build_and_run.sh
1010-```1616+### checklist before it's usable
1717+* [ ] Ingesting content from the firehose (using Jetstream)
1818+* [ ] Creating the lexicon documents and validating content that comes in from the firehose
1919+* [ ] Indexing valid content from the firehose into a database
2020+* [ ] Handling OAuth authentication (public OAuth for the moment)
2121+* [ ] Creating responses to API calls
2222+* [ ] Create records through the API
2323+* [ ] Interact with the frontend
-30
backend/analysis_options.yaml
···11-# This file configures the static analysis results for your project (errors,
22-# warnings, and lints).
33-#
44-# This enables the 'recommended' set of lints from `package:lints`.
55-# This set helps identify many issues that may lead to problems when running
66-# or consuming Dart code, and enforces writing Dart using a single, idiomatic
77-# style and format.
88-#
99-# If you want a smaller set of lints you can change this to specify
1010-# 'package:lints/core.yaml'. These are just the most critical lints
1111-# (the recommended set includes the core lints).
1212-# The core lints are also what is used by pub.dev for scoring packages.
1313-1414-include: package:lints/recommended.yaml
1515-1616-# Uncomment the following section to specify additional rules.
1717-1818-# linter:
1919-# rules:
2020-# - camel_case_types
2121-2222-# analyzer:
2323-# exclude:
2424-# - path/to/excluded/files/**
2525-2626-# For more information about the core and recommended set of lints, see
2727-# https://dart.dev/go/core-lints
2828-2929-# For additional information about configuring this file, see
3030-# https://dart.dev/guides/language/analysis-options
-25
backend/bin/clippr.dart
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-import 'package:clippr/config/config.dart' show Config;
88-import 'package:clippr/config/pubspec.dart' show ClipprPubspec;
99-import 'package:clippr/server/logger.dart' show Logger;
1010-import 'package:clippr/server/server.dart';
1111-1212-void main(List<String> arguments) {
1313- ClipprPubspec(); // initialize pubspec
1414- Logger(); // initialize logger
1515- Logger.logInfo("${ClipprPubspec.getName()} ${ClipprPubspec.getVersion()}");
1616- Logger.logInfo("Initializing config...");
1717- Config(); // initialize config
1818- Logger.logInfo("Initializing database at ${Config.getDatabaseName()}...");
1919- launchDatabase();
2020- Logger.logInfo("Initializing firehose...");
2121- launchFirehose();
2222- Logger.logInfo("Starting server at ${Config.getHostname()}:${Config.getPort()}");
2323- launchWebServer();
2424- Logger.logInfo("Server launched!");
2525-}
···11+## This is a configuration file for Clippr.
22+## Please copy to "config.example.toml" before starting the server, otherwise it will not start.
33+## Modify as necessary.
44+55+hostname = "localhost"
66+port = 9090
77+88+## How the SQLite database is stored.
99+[database]
1010+name = "clippr.db"
1111+1212+## How the server interacts with the ATproto network.
1313+[network]
1414+firehose = "jetstream1.us-east.bsky.network"
-16
backend/config.example.yaml
···11-## Clippr-BE config
22-## If this is not copied to "config.yaml", the server will not launch.
33-## Modify what you need!
44-55-## Configure where the server opens up to.
66-server:
77- hostname: localhost
88- port: 9090
99- database_name: "clippr.db"
1010-1111-## General AT Protocol settings (relay communication for the moment)
1212-network:
1313- firehose_provider: "jetstream1.us-east.bsky.network"
1414-1515-## OAuth settings, for authentication
1616-oauth:
···11+import js from "@eslint/js";
22+import globals from "globals";
33+import tseslint from "typescript-eslint";
44+import {defineConfig} from "eslint/config";
55+66+77+export default defineConfig([
88+ {files: ["**/*.{js,mjs,cjs,ts,mts,cts}"], plugins: {js}, extends: ["js/recommended"]},
99+ {files: ["**/*.{js,mjs,cjs,ts,mts,cts}"], languageOptions: {globals: globals.browser}},
1010+ tseslint.configs.recommended,
1111+ tseslint.configs.stylistic
1212+]);
-7
backend/lib/auth/create_session.dart
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-// Blank for now...
-10
backend/lib/auth/oauth.dart
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-class OAuth {
88- late final String clientId;
99- late final String clientName;
1010-}
-53
backend/lib/clip_id/clip_id.dart
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-/// A class that implements the ``ClipID`` system that we use to create valid, de-duplicated record keys for Clips.
88-///
99-/// To provide an example, say we want to convert the link ``https://blog.example.com/~archives/hello-world.html`` into a ClipID.
1010-///
1111-/// To do this, first we will remove the scheme from the link. If you plan to support non-HTTP schemes such as Gemini,
1212-/// you would precede the hostname with the scheme (Example: ``gemini:blog.example.com``).
1313-///
1414-/// Separating the hostname from the file path is simple, simply divide the two with a colon
1515-/// (``blog.example.com:/~archives/hello-world.html``).
1616-/// If there is no file path (i.e. the link is an index page on root) make the link path a tilde.
1717-///
1818-/// We replace the slash separators with tildes. To account for paths with tildes, simply add an underscore before one (``_~archives``).
1919-///
2020-/// The final ClipID should be `blog.example.com:_~archives~hello-world.html`.
2121-/// This is not a perfect system and could be replaced or modified at any moment when issues arrive,
2222-/// however we believe it is functional for the moment.
2323-class ClipID {
2424- /// The scheme of the ClipID.
2525- String? scheme;
2626- /// The hostname of the ClipID.
2727- String? hostname;
2828- /// The filepath of the ClipID.
2929- String? path;
3030- /// The segments of the filepath.
3131- List<String>? pathSegments;
3232-3333- ClipID(Uri url) {
3434- scheme = url.scheme;
3535- hostname = url.host;
3636- if (url.hasEmptyPath || url.pathSegments.isEmpty) {
3737- path = "~";
3838- pathSegments = ["~"];
3939- } else {
4040- path = url.path;
4141- path = path?.replaceAll("~", "_~");
4242- path = path?.replaceFirst("/", "");
4343- path = path?.replaceAll("/", "~");
4444- pathSegments = path?.split(RegExp(r"(?<!_)~"));
4545- }
4646- }
4747-4848- /// Prints out the full ClipID.
4949- @override
5050- String toString() {
5151- return "$scheme:$hostname:$path";
5252- }
5353-}
-46
backend/lib/config/config.dart
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-import 'dart:io';
88-import 'package:clippr/server/logger.dart' show Logger;
99-import 'package:yaml/yaml.dart';
1010-1111-class Config {
1212- static Map? yamlDoc;
1313-1414- Config() {
1515- File file = File('config.yaml');
1616- if (file.existsSync()) {
1717- yamlDoc = loadYaml(file.readAsStringSync()) as Map;
1818- return;
1919- }
2020-2121- file = File('config.yml');
2222- if (file.existsSync()) {
2323- yamlDoc = loadYaml(file.readAsStringSync()) as Map;
2424- return;
2525- }
2626-2727- Logger.logSevere("Failed to read config.yaml");
2828- exit(1);
2929- }
3030-3131- static int getPort() {
3232- return yamlDoc?['server']['port'];
3333- }
3434-3535- static String getHostname() {
3636- return yamlDoc?['server']['hostname'];
3737- }
3838-3939- static String getDatabaseName() {
4040- return yamlDoc?['server']['database_name'];
4141- }
4242-4343- static String getFirehoseProvider() {
4444- return yamlDoc?['network']['firehose_provider'];
4545- }
4646-}
-26
backend/lib/config/pubspec.dart
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-import 'dart:io';
88-99-import 'package:pubspec_parse/pubspec_parse.dart';
1010-1111-class ClipprPubspec {
1212- static late Pubspec pubspec;
1313-1414- ClipprPubspec() {
1515- final pubspecFile = File("pubspec.yaml").readAsStringSync();
1616- pubspec = Pubspec.parse(pubspecFile);
1717- }
1818-1919- static String getVersion() {
2020- return pubspec.version.toString();
2121- }
2222-2323- static String getName() {
2424- return pubspec.name;
2525- }
2626-}
-89
backend/lib/db/database.dart
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-import 'dart:convert';
88-import 'dart:io';
99-1010-import 'package:atproto/atproto.dart';
1111-import 'package:clippr/config/config.dart';
1212-import 'package:drift/drift.dart';
1313-import 'package:drift/native.dart';
1414-1515-part 'database.g.dart';
1616-1717-class Clips extends Table {
1818- DateTimeColumn get timestamp =>
1919- dateTime().named("time_us").withDefault(currentDateAndTime)();
2020- TextColumn get did => text()();
2121- TextColumn get recordKey => text().named("rkey")();
2222- TextColumn get url => text()();
2323- TextColumn get title => text()();
2424- TextColumn get description => text()();
2525- TextColumn get notes => text().nullable()();
2626- TextColumn get tags => text().map(const StrongRefConverter()).nullable()();
2727- BoolColumn get unlisted => boolean()();
2828- BoolColumn get unread => boolean().nullable()();
2929- TextColumn get languages =>
3030- text().map(const StringArrayConverter()).nullable()();
3131- DateTimeColumn get createdAt => dateTime()();
3232- @override
3333- Set<Column<Object>> get primaryKey => {timestamp};
3434-}
3535-3636-class Tags extends Table {
3737- DateTimeColumn get timestamp =>
3838- dateTime().named("time_us").withDefault(currentDateAndTime)();
3939- TextColumn get did => text()();
4040- TextColumn get recordKey => text().named('rkey')();
4141- TextColumn get name => text()();
4242- TextColumn get color => text().nullable()();
4343- DateTimeColumn get createdAt => dateTime()();
4444- @override
4545- Set<Column<Object>> get primaryKey => {timestamp};
4646-}
4747-4848-@DriftDatabase(tables: [Clips])
4949-class AppDatabase extends _$AppDatabase {
5050- AppDatabase([QueryExecutor? executor]) : super(executor ?? _openConnection());
5151-5252- @override
5353- int get schemaVersion => 1;
5454-5555- static QueryExecutor _openConnection() {
5656- return NativeDatabase.createInBackground(File(Config.getDatabaseName()));
5757- }
5858-}
5959-6060-/// Convert an array of strings to and from JSON for Drift.
6161-class StringArrayConverter extends TypeConverter<List<String>, String> {
6262- const StringArrayConverter();
6363-6464- @override
6565- List<String> fromSql(String fromDb) {
6666- return (jsonDecode(fromDb) as List<dynamic>).cast<String>();
6767- }
6868-6969- @override
7070- String toSql(List<String> value) {
7171- return jsonEncode(value);
7272- }
7373-}
7474-7575-/// Convert [StrongRef] to and from JSON for Drift.
7676-class StrongRefConverter extends TypeConverter<StrongRef, String> {
7777- const StrongRefConverter();
7878-7979- @override
8080- StrongRef fromSql(String fromDb) {
8181- final map = jsonDecode(fromDb);
8282- return StrongRef.fromJson(map);
8383- }
8484-8585- @override
8686- String toSql(StrongRef value) {
8787- return jsonEncode(value.toJson());
8888- }
8989-}
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-import 'dart:async';
88-import 'dart:convert';
99-1010-import 'package:clippr/server/logger.dart';
1111-import 'package:web_socket_channel/web_socket_channel.dart';
1212-1313-/// This class creates a WebSocket connection to a Jetstream-based firehose.
1414-///
1515-/// All instances operate their own separate connection to the firehose.
1616-/// As such, new instances should be created as little as possible.
1717-class Firehose {
1818- /// [Uri] containing a link to the firehose's WebSocket subscription link.
1919- static late final Uri firehoseProvider;
2020- /// Raw WebSocket stream
2121- static late final WebSocketChannel _channel;
2222- /// Controllable version of stream
2323- static final StreamController<Map<String, dynamic>> _messageController =
2424- StreamController.broadcast();
2525- /// Stream of messages to access from other places
2626- static Stream<Map<String, dynamic>> get messageStream => _messageController.stream;
2727-2828- /// Create a new [Firehose] instance using a provided URL string.
2929- Firehose(String firehoseUrl) {
3030- if (firehoseUrl.startsWith("wss://") || firehoseUrl.startsWith("ws://")) {
3131- firehoseUrl = firehoseUrl.replaceFirst("ws://", "");
3232- firehoseUrl = firehoseUrl.replaceFirst("wss://", "");
3333- }
3434- if (firehoseUrl.endsWith("/subscribe")) {
3535- firehoseUrl = firehoseUrl.replaceFirst("/subscribe", "");
3636- }
3737- // This is hardcoded for the moment. This is not ideal and we should end up
3838- // splitting this code into its own separate package with tests and custom
3939- // query parameters but right now we just need to fetch our stuff.
4040- firehoseProvider = Uri.parse(
4141- 'wss://$firehoseUrl/subscribe?wantedCollections=social.clippr.*',
4242- );
4343- }
4444-4545- /// Connect to the firehose.
4646- ///
4747- /// TODO: Implement support for the query parameters.
4848- Future<void> connect() async {
4949- _channel = WebSocketChannel.connect(firehoseProvider);
5050- await _channel.ready;
5151-5252- _channel.stream.listen( // Pass raw WebSocket stream to controller
5353- (message) {
5454- try {
5555- final jsonMessage = jsonDecode(message);
5656- _messageController.add(jsonMessage);
5757- } catch (e) {
5858- print('Invalid JSON: $e');
5959- }
6060- },
6161- onDone: () {
6262- Logger.logSevere('WebSocket connection closed by firehose');
6363- _messageController.close();
6464- },
6565- onError: (error) {
6666- Logger.logSevere('WebSocket error: $error');
6767- _messageController.close();
6868- },
6969- );
7070- }
7171-7272- /// Disconnect from the firehose.
7373- void disconnect() {
7474- _channel.sink.close(1000, "Normal closure");
7575- _messageController.close();
7676- }
7777-}
···11+import {Hono} from 'hono'
22+import {serveStatic} from "hono/bun";
33+44+const app = new Hono();
55+66+app.use('/static/*', serveStatic({root: './'}));
77+app.get('/', serveStatic({path: './static/index.html'}));
88+app.notFound((c) => {
99+ return c.text('404 Not Found', 404);
1010+})
1111+1212+export default app;
+9
backend/src/server.ts
···11+import {Hono} from "hono";
22+import misc from "./routes/misc";
33+44+const app = new Hono();
55+66+// Link all routes up
77+app.route('/', misc);
88+99+export default app;
···11-/*
22- * clippr: a social bookmarking service for the AT Protocol
33- * Copyright (c) 2025 clippr contributors.
44- * SPDX-License-Identifier: AGPL-3.0-only
55- */
66-77-import 'package:test/test.dart';
88-import 'package:clippr/clip_id/clip_id.dart';
99-1010-void main() {
1111- late Uri urlWithEmptyPath;
1212- late Uri urlWithPath;
1313- late Uri urlWithTildePath;
1414- setUp(() {
1515- urlWithEmptyPath = Uri.parse("https://example.com/");
1616- urlWithPath = Uri.parse("http://testers.org/example/path.html");
1717- urlWithTildePath = Uri.parse("gemini://bell.labs/~dmr/unix.html");
1818- });
1919-2020- group('URL with empty path', () {
2121- test('Example URL with empty path: scheme is correct', () {
2222- var id = ClipID(urlWithEmptyPath);
2323- expect(id.scheme, equals("https"));
2424- });
2525- test('Example URL with empty path: hostname is correct', () {
2626- var id = ClipID(urlWithEmptyPath);
2727- expect(id.hostname, equals("example.com"));
2828- });
2929- test('Example URL with empty path: path is correct', () {
3030- var id = ClipID(urlWithEmptyPath);
3131- expect(id.path, equals("~"));
3232- });
3333- test('Example URL with empty path: path segments are correct', () {
3434- var id = ClipID(urlWithEmptyPath);
3535- expect(id.pathSegments, equals(["~"]));
3636- });
3737- test('Example URL with empty path: complete ID is correct', () {
3838- var id = ClipID(urlWithEmptyPath);
3939- expect(id.toString(), equals("https:example.com:~"));
4040- });
4141- });
4242-4343- group('Example URL with path', () {
4444- test('Example URL with path: scheme is correct', () {
4545- var id = ClipID(urlWithPath);
4646- expect(id.scheme, equals("http"));
4747- });
4848-4949- test('Example URL with path: hostname is correct', () {
5050- var id = ClipID(urlWithPath);
5151- expect(id.hostname, equals("testers.org"));
5252- });
5353-5454- test('Example URL with path: path is correct', () {
5555- var id = ClipID(urlWithPath);
5656- expect(id.path, equals("example~path.html"));
5757- });
5858-5959- test('Example URL with path: path segments are correct', () {
6060- var id = ClipID(urlWithPath);
6161- expect(id.pathSegments, equals(["example", "path.html"]));
6262- });
6363-6464- test('Example URL with path: complete ID is correct', () {
6565- var id = ClipID(urlWithPath);
6666- expect(id.toString(), equals("http:testers.org:example~path.html"));
6767- });
6868- });
6969-7070- group('Example URL with tilde-included path', () {
7171- test('Example URL with tilde path: scheme is correct', () {
7272- var id = ClipID(urlWithTildePath);
7373- expect(id.scheme, equals("gemini"));
7474- });
7575-7676- test('Example URL with tilde path: hostname is correct', () {
7777- var id = ClipID(urlWithTildePath);
7878- expect(id.hostname, equals("bell.labs"));
7979- });
8080-8181- test('Example URL with tilde path: path is correct', () {
8282- var id = ClipID(urlWithTildePath);
8383- expect(id.path, equals("_~dmr~unix.html"));
8484- });
8585-8686- test('Example URL with tilde path: path segments are correct', () {
8787- var id = ClipID(urlWithTildePath);
8888- expect(id.pathSegments, equals(["_~dmr", "unix.html"]));
8989- });
9090-9191- test('Example URL with tilde path: complete ID is correct', () {
9292- var id = ClipID(urlWithTildePath);
9393- expect(id.toString(), equals("gemini:bell.labs:_~dmr~unix.html"));
9494- });
9595- });
9696-}
-8
backend/tools/build_and_run.sh
···11-#!/bin/sh
22-33-# clippr: a social bookmarking service for the AT Protocol
44-# Copyright (c) 2025 clippr contributors.
55-# SPDX-License-Identifier: AGPL-3.0-only
66-77-dart run build_runner build
88-dart run bin/clippr.dart