openstatus.dev
Openstatus
www.openstatus.dev
feat: audit logs (#479)
* feat: tb audit log
* chore: add example
* fix: pipe order by
* feat: add generic metadata schema
* chore: audit example in server
* chore: more zod magic
* wip: audit log
* fix: import and descriptions
* chore: improve audit log
* chore: include types
* fix: typo
* chore: small improvements
* fix: index
authored by
Maximilian Kaske
and committed by
GitHub
74a234dc
432f4867
+480
-23
+9
apps/server/src/checker/alerting.ts
+9
apps/server/src/checker/alerting.ts
···
7
7
import { flyRegionsDict } from "@openstatus/utils";
8
8
9
9
import { env } from "../env";
10
+
import { checkerAudit } from "../utils/audit-log";
10
11
import { providerToFunction } from "./utils";
11
12
12
13
export const triggerAlerting = async ({
···
43
44
statusCode,
44
45
message,
45
46
});
47
+
// ALPHA
48
+
await checkerAudit.publishAuditLog({
49
+
id: `monitor:${monitorId}`,
50
+
action: "notification.sent",
51
+
targets: [{ id: monitorId, type: "monitor" }],
52
+
metadata: { provider: notif.notification.provider },
53
+
});
54
+
//
46
55
}
47
56
};
48
57
+4
-18
apps/server/src/checker/checker.ts
+4
-18
apps/server/src/checker/checker.ts
···
1
-
import { env } from "../env";
2
-
import { triggerAlerting, upsertMonitorStatus } from "./alerting";
1
+
import { handleMonitorFailed, handleMonitorRecovered } from "./monitor-handler";
3
2
import type { PublishPingType } from "./ping";
4
3
import { pingEndpoint, publishPing } from "./ping";
5
4
import type { Payload } from "./schema";
···
76
75
message: undefined,
77
76
});
78
77
if (data?.status === "error") {
79
-
await upsertMonitorStatus({
80
-
monitorId: data.monitorId,
81
-
status: "active",
82
-
});
78
+
handleMonitorRecovered(data, res);
83
79
}
84
80
} else {
85
81
if (retry < 2) {
···
96
92
payload: data,
97
93
latency,
98
94
statusCode: res?.status,
99
-
message: message,
95
+
message,
100
96
});
101
-
102
97
if (data?.status === "active") {
103
-
await upsertMonitorStatus({
104
-
monitorId: data.monitorId,
105
-
status: "error",
106
-
});
107
-
await triggerAlerting({
108
-
monitorId: data.monitorId,
109
-
region: env.FLY_REGION,
110
-
statusCode: res?.status,
111
-
message,
112
-
});
98
+
handleMonitorFailed(data, res, message);
113
99
}
114
100
}
115
101
}
+48
apps/server/src/checker/monitor-handler.ts
+48
apps/server/src/checker/monitor-handler.ts
···
1
+
import { env } from "../env";
2
+
import { checkerAudit } from "../utils/audit-log";
3
+
import { triggerAlerting, upsertMonitorStatus } from "./alerting";
4
+
import type { Payload } from "./schema";
5
+
6
+
export async function handleMonitorRecovered(data: Payload, res: Response) {
7
+
await upsertMonitorStatus({
8
+
monitorId: data.monitorId,
9
+
status: "active",
10
+
});
11
+
// ALPHA
12
+
await checkerAudit.publishAuditLog({
13
+
id: `monitor:${data.monitorId}`,
14
+
action: "monitor.recovered",
15
+
targets: [{ id: data.monitorId, type: "monitor" }],
16
+
metadata: { region: env.FLY_REGION, statusCode: res.status },
17
+
});
18
+
//
19
+
}
20
+
21
+
export async function handleMonitorFailed(
22
+
data: Payload,
23
+
res: Response | null,
24
+
message?: string,
25
+
) {
26
+
await upsertMonitorStatus({
27
+
monitorId: data.monitorId,
28
+
status: "error",
29
+
});
30
+
// ALPHA
31
+
await checkerAudit.publishAuditLog({
32
+
id: `monitor:${data.monitorId}`,
33
+
action: "monitor.failed",
34
+
targets: [{ id: data.monitorId, type: "monitor" }],
35
+
metadata: {
36
+
region: env.FLY_REGION,
37
+
statusCode: res?.status,
38
+
message,
39
+
},
40
+
});
41
+
//
42
+
await triggerAlerting({
43
+
monitorId: data.monitorId,
44
+
region: env.FLY_REGION,
45
+
statusCode: res?.status,
46
+
message,
47
+
});
48
+
}
+7
apps/server/src/utils/audit-log.ts
+7
apps/server/src/utils/audit-log.ts
+13
packages/tinybird/datasources/audit_log.datasource
+13
packages/tinybird/datasources/audit_log.datasource
···
1
+
VERSION 0
2
+
3
+
SCHEMA >
4
+
`action` String `json:$.action`,
5
+
`actor` String `json:$.actor`,
6
+
`id` String `json:$.id`,
7
+
`targets` Nullable(String) `json:$.targets`,
8
+
`metadata` Nullable(String) `json:$.metadata`,
9
+
`timestamp` Int64 `json:$.timestamp`,
10
+
`version` Int16 `json:$.version`
11
+
12
+
ENGINE "MergeTree"
13
+
ENGINE_SORTING_KEY "id, timestamp, action"
+1
-1
packages/tinybird/package.json
+1
-1
packages/tinybird/package.json
+8
packages/tinybird/pipes/endpoint_audit_log.pipe
+8
packages/tinybird/pipes/endpoint_audit_log.pipe
+99
packages/tinybird/src/audit-log/README.md
+99
packages/tinybird/src/audit-log/README.md
···
1
+
## Motivation
2
+
3
+
We want to track every change made for the `incident` and `monitor`. Therefore,
4
+
it requires us to build some audit log / event sourcing foundation.
5
+
6
+
The `Event` is what the data type stored within [Tinybird](https://tinybird.co).
7
+
It has basic props that every event includes, as well as a `metadata` prop that
8
+
can be used to store additional informations.
9
+
10
+
```ts
11
+
export type Event = {
12
+
/**
13
+
* Unique identifier for the event.
14
+
*/
15
+
id: string;
16
+
17
+
/**
18
+
* The actor that triggered the event.
19
+
* @default { id: "", name: "system" }
20
+
* @example { id: "1", name: "mxkaske" }
21
+
*/
22
+
actor?: {
23
+
id: string;
24
+
name: string;
25
+
};
26
+
27
+
/**
28
+
* The ressources affected by the action taken.
29
+
* @example [{ id: "1", name: "monitor" }]
30
+
*/
31
+
targets?: {
32
+
id: string;
33
+
name: string;
34
+
}[];
35
+
36
+
/**
37
+
* The action that was triggered.
38
+
* @example monitor.down | incident.create
39
+
*/
40
+
action: string;
41
+
42
+
/**
43
+
* The timestamp of the event in milliseconds since epoch UTC.
44
+
* @default Date.now()
45
+
*/
46
+
timestamp?: number;
47
+
48
+
/**
49
+
* The version of the event. Should be incremented on each update.
50
+
* @default 1
51
+
*/
52
+
version?: number;
53
+
54
+
/**
55
+
* Metadata for the event. Defined via zod schema.
56
+
*/
57
+
metadata?: unknown;
58
+
};
59
+
```
60
+
61
+
The objects are parsed and stored as string via
62
+
`schema.transform(val => JSON.stringify(val))` and transformed back into an
63
+
object before parsing via `z.preprocess(val => JSON.parse(val), schema)`.
64
+
65
+
## Example
66
+
67
+
```ts
68
+
const tb = new Tinybird({ token: process.env.TINY_BIRD_API_KEY || "" });
69
+
70
+
const auditLog = new AuditLog({ tb });
71
+
72
+
await auditLog.publishAuditLog({
73
+
id: "monitor:1",
74
+
action: "monitor.down",
75
+
targets: [{ id: "1", type: "monitor" }], // not mandatory, but could be useful later on
76
+
metadata: { region: "gru", statusCode: 400, message: "timeout" },
77
+
});
78
+
79
+
await auditLog.getAuditLog({ event_id: "monitor:1" });
80
+
```
81
+
82
+
## Inspiration
83
+
84
+
- WorkOS [Audit Logs](https://workos.com/docs/audit-logs)
85
+
86
+
## Tinybird
87
+
88
+
Push the pipe and datasource to tinybird:
89
+
90
+
```
91
+
tb push datasources/audit_log.datasource
92
+
tb push pipes/endpoint_audit_log.pipe
93
+
```
94
+
95
+
---
96
+
97
+
### Possible extention
98
+
99
+
> TODO: Remove `Nullable` from `targets` to better index and query it.
+47
packages/tinybird/src/audit-log/action-schema.ts
+47
packages/tinybird/src/audit-log/action-schema.ts
···
1
+
import { z } from "zod";
2
+
3
+
/**
4
+
* The schema for the monitor.recovered action.
5
+
* It represents the event when a monitor has recovered from a failure.
6
+
*/
7
+
export const monitorRecoveredSchema = z.object({
8
+
action: z.literal("monitor.recovered"),
9
+
metadata: z.object({ region: z.string(), statusCode: z.number() }),
10
+
});
11
+
12
+
/**
13
+
* The schema for the monitor.failed action.
14
+
* It represents the event when a monitor has failed.
15
+
*/
16
+
export const monitorFailedSchema = z.object({
17
+
action: z.literal("monitor.failed"),
18
+
metadata: z.object({
19
+
region: z.string(),
20
+
statusCode: z.number().optional(),
21
+
message: z.string().optional(),
22
+
}),
23
+
});
24
+
25
+
/**
26
+
* The schema for the notification.send action.
27
+
*
28
+
*/
29
+
export const notificationSentSchema = z.object({
30
+
action: z.literal("notification.sent"),
31
+
// we could use the notificationProviderSchema for more type safety
32
+
metadata: z.object({ provider: z.string() }),
33
+
});
34
+
35
+
// TODO: update schemas with correct metadata and description
36
+
37
+
export const incidentCreatedSchema = z.object({
38
+
action: z.literal("incident.created"),
39
+
metadata: z.object({}), // tbd
40
+
});
41
+
42
+
export const incidentResolvedSchema = z.object({
43
+
action: z.literal("incident.resolved"),
44
+
metadata: z.object({}), // tbd
45
+
});
46
+
47
+
// ...
+67
packages/tinybird/src/audit-log/action-validation.ts
+67
packages/tinybird/src/audit-log/action-validation.ts
···
1
+
import { z } from "zod";
2
+
3
+
import {
4
+
monitorFailedSchema,
5
+
monitorRecoveredSchema,
6
+
notificationSentSchema,
7
+
} from "./action-schema";
8
+
import { ingestBaseEventSchema, pipeBaseResponseData } from "./base-validation";
9
+
10
+
/**
11
+
* The schema for the event object.
12
+
* It extends the base schema. It uses the `discriminatedUnion` method for faster
13
+
* evaluation to determine which schema to be used to parse the input.
14
+
* It also transforms the metadata object into a string.
15
+
*
16
+
* @todo: whenever a new action is added, it should be included to the discriminatedUnion
17
+
*/
18
+
export const ingestActionEventSchema = z
19
+
.intersection(
20
+
// Unfortunately, the array cannot be dynamic, otherwise could be added to the Client
21
+
// and made available to devs as library
22
+
z.discriminatedUnion("action", [
23
+
monitorRecoveredSchema,
24
+
monitorFailedSchema,
25
+
notificationSentSchema,
26
+
]),
27
+
ingestBaseEventSchema,
28
+
)
29
+
.transform((val) => ({
30
+
...val,
31
+
metadata: JSON.stringify(val.metadata),
32
+
}));
33
+
34
+
/**
35
+
* The schema for the response object.
36
+
* It extends the base schema. It uses the `discriminatedUnion` method for faster
37
+
* evaluation to determine which schema to be used to parse the input.
38
+
* It also preprocesses the metadata string into the correct schema object.
39
+
*
40
+
* @todo: whenever a new action is added, it should be included to the discriminatedUnion
41
+
*/
42
+
export const pipeActionResponseData = z.intersection(
43
+
z.discriminatedUnion("action", [
44
+
monitorRecoveredSchema.extend({
45
+
metadata: z.preprocess(
46
+
(val) => JSON.parse(String(val)),
47
+
monitorRecoveredSchema.shape.metadata,
48
+
),
49
+
}),
50
+
monitorFailedSchema.extend({
51
+
metadata: z.preprocess(
52
+
(val) => JSON.parse(String(val)),
53
+
monitorFailedSchema.shape.metadata,
54
+
),
55
+
}),
56
+
notificationSentSchema.extend({
57
+
metadata: z.preprocess(
58
+
(val) => JSON.parse(String(val)),
59
+
notificationSentSchema.shape.metadata,
60
+
),
61
+
}),
62
+
]),
63
+
pipeBaseResponseData,
64
+
);
65
+
66
+
export type IngestActionEvent = z.infer<typeof ingestActionEventSchema>;
67
+
export type PipeActionResponseData = z.infer<typeof pipeActionResponseData>;
+95
packages/tinybird/src/audit-log/base-validation.ts
+95
packages/tinybird/src/audit-log/base-validation.ts
···
1
+
import { z } from "zod";
2
+
3
+
/**
4
+
* The base schema for every event, used to validate it's structure
5
+
* on datasource ingestion and pipe retrieval.
6
+
*/
7
+
export const baseSchema = z.object({
8
+
id: z.string().min(1), // DISCUSS: we could use the `${targets.type}:${targets.id}` format automatic generation
9
+
action: z.string(),
10
+
// REMINDER: do not use .default(Date.now()), it will be evaluated only once
11
+
timestamp: z
12
+
.number()
13
+
.int()
14
+
.optional()
15
+
.transform((val) => val || Date.now()),
16
+
version: z.number().int().default(1),
17
+
});
18
+
19
+
/**
20
+
* The schema for the actor type.
21
+
* It represents the type of the user that triggered the event.
22
+
*/
23
+
export const actorTypeSchema = z.enum(["user", "system"]);
24
+
25
+
/**
26
+
* The schema for the actor object.
27
+
* It represents the user that triggered the event.
28
+
*/
29
+
export const actorSchema = z
30
+
.object({
31
+
id: z.string(),
32
+
type: actorTypeSchema,
33
+
})
34
+
.default({
35
+
id: "server",
36
+
type: "system",
37
+
});
38
+
39
+
/**
40
+
* The schema for the target type.
41
+
* It represents the type of the target that is affected by the event.
42
+
*/
43
+
export const targetTypeSchema = z.enum([
44
+
"monitor",
45
+
"page",
46
+
"incident",
47
+
"user",
48
+
"notification",
49
+
"organization",
50
+
]);
51
+
52
+
/**
53
+
* The schema for the targets object.
54
+
* It represents the targets that are affected by the event.
55
+
*/
56
+
export const targetsSchema = z
57
+
.object({
58
+
id: z.string(),
59
+
type: targetTypeSchema,
60
+
})
61
+
.array()
62
+
.optional();
63
+
64
+
/**
65
+
* The schema for the event object.
66
+
* It extends the base schema and transforms the actor, targets
67
+
* objects into strings.
68
+
*/
69
+
export const ingestBaseEventSchema = baseSchema.extend({
70
+
actor: actorSchema.transform((val) => JSON.stringify(val)),
71
+
targets: targetsSchema.transform((val) => JSON.stringify(val)),
72
+
});
73
+
74
+
/**
75
+
* The schema for the response object.
76
+
* It extends the base schema and transforms the actor, targets
77
+
* back into typed objects.
78
+
*/
79
+
export const pipeBaseResponseData = baseSchema.extend({
80
+
actor: z.preprocess((val) => JSON.parse(String(val)), actorSchema),
81
+
targets: z.preprocess(
82
+
(val) => (val ? JSON.parse(String(val)) : undefined),
83
+
targetsSchema,
84
+
),
85
+
});
86
+
87
+
/**
88
+
* The schema for the parameters object.
89
+
* It represents the parameters that are passed to the pipe.
90
+
*/
91
+
export const pipeParameterData = z.object({ event_id: z.string().min(1) });
92
+
93
+
export type PipeParameterData = z.infer<typeof pipeParameterData>;
94
+
export type PipeBaseResponseData = z.infer<typeof pipeBaseResponseData>;
95
+
export type IngestBaseEvent = z.infer<typeof ingestBaseEventSchema>;
+30
packages/tinybird/src/audit-log/client.ts
+30
packages/tinybird/src/audit-log/client.ts
···
1
+
import type { Tinybird } from "@chronark/zod-bird";
2
+
3
+
import {
4
+
ingestActionEventSchema,
5
+
pipeActionResponseData,
6
+
} from "./action-validation";
7
+
import { pipeParameterData } from "./base-validation";
8
+
9
+
export class AuditLog {
10
+
private readonly tb: Tinybird;
11
+
12
+
constructor(opts: { tb: Tinybird }) {
13
+
this.tb = opts.tb;
14
+
}
15
+
16
+
get publishAuditLog() {
17
+
return this.tb.buildIngestEndpoint({
18
+
datasource: "audit_log__v0",
19
+
event: ingestActionEventSchema,
20
+
});
21
+
}
22
+
23
+
get getAuditLog() {
24
+
return this.tb.buildPipe({
25
+
pipe: "endpoint_audit_log__v0",
26
+
parameters: pipeParameterData,
27
+
data: pipeActionResponseData,
28
+
});
29
+
}
30
+
}
+46
packages/tinybird/src/audit-log/examples.ts
+46
packages/tinybird/src/audit-log/examples.ts
···
1
+
/* eslint-disable @typescript-eslint/no-unused-vars */
2
+
import { Tinybird } from "@chronark/zod-bird";
3
+
4
+
import { AuditLog } from "./client";
5
+
6
+
const tb = new Tinybird({ token: process.env.TINY_BIRD_API_KEY || "" });
7
+
8
+
const auditLog = new AuditLog({ tb });
9
+
10
+
async function seed() {
11
+
await auditLog.publishAuditLog({
12
+
id: "monitor:2",
13
+
action: "monitor.failed",
14
+
targets: [{ id: "2", type: "monitor" }],
15
+
metadata: { region: "gru", statusCode: 500, message: "timeout" },
16
+
});
17
+
await auditLog.publishAuditLog({
18
+
id: "monitor:1",
19
+
action: "monitor.recovered",
20
+
targets: [{ id: "1", type: "monitor" }],
21
+
metadata: { region: "gru", statusCode: 200 },
22
+
});
23
+
await auditLog.publishAuditLog({
24
+
id: "user:1",
25
+
actor: {
26
+
type: "user",
27
+
id: "1",
28
+
},
29
+
targets: [{ id: "1", type: "user" }],
30
+
action: "notification.sent",
31
+
metadata: { provider: "email" },
32
+
});
33
+
}
34
+
35
+
async function history() {
36
+
return await auditLog.getAuditLog({ event_id: "user:1" });
37
+
}
38
+
39
+
// seed();
40
+
// const all = await history();
41
+
// console.log(all);
42
+
// const first = all.data[0];
43
+
44
+
// if (first.action === "monitor.failed") {
45
+
// first.metadata.message;
46
+
// }
+1
packages/tinybird/src/audit-log/index.ts
+1
packages/tinybird/src/audit-log/index.ts
···
1
+
export * from "./client";
+1
packages/tinybird/src/index.ts
+1
packages/tinybird/src/index.ts
+4
-4
pnpm-lock.yaml
+4
-4
pnpm-lock.yaml
···
745
745
packages/tinybird:
746
746
dependencies:
747
747
'@chronark/zod-bird':
748
-
specifier: 0.2.2
749
-
version: 0.2.2
748
+
specifier: 0.3.1
749
+
version: 0.3.1
750
750
zod:
751
751
specifier: 3.22.2
752
752
version: 3.22.2
···
1180
1180
'@babel/helper-validator-identifier': 7.22.20
1181
1181
to-fast-properties: 2.0.0
1182
1182
1183
-
/@chronark/zod-bird@0.2.2:
1184
-
resolution: {integrity: sha512-0mmiiw4dny1aiEOmawsIJUaYW16vhWRuDGsDa62GfFhdD3F7hHrTi3lWNWYCVq2KsCS+CYwMj8nkdFuOnejCTA==}
1183
+
/@chronark/zod-bird@0.3.1:
1184
+
resolution: {integrity: sha512-4PNJx41m37Psk/3XAxdXxMb9VGAep43/puiwxAyFWYzIoQ1kq3Cwh1sA51LkTFSVHm+B2peMmW+p3/+oCHH9zg==}
1185
1185
dependencies:
1186
1186
zod: 3.22.2
1187
1187
dev: false