openstatus.dev
Openstatus
www.openstatus.dev
chore: export script (#1478)
* chore: export script
* fix: format and timestamp
authored by
Maximilian Kaske
and committed by
GitHub
61077cfb
6d1c9544
+342
+115
apps/dashboard/src/scripts/README.md
+115
apps/dashboard/src/scripts/README.md
···
1
+
# Export Blog Post Metrics Script
2
+
3
+
This script exports monitor metrics data from OpenStatus for use in blog posts and documentation.
4
+
5
+
## Overview
6
+
7
+
The script fetches monitor data directly from the database and Tinybird analytics, then exports it to a JSON file that can be used for visualizations in blog posts.
8
+
9
+
**Features:**
10
+
- Fetches metrics from both regular regions and private locations
11
+
- Automatically combines public regions with private location data
12
+
- Supports both HTTP and TCP monitors
13
+
14
+
## Configuration
15
+
16
+
Edit the constants at the top of `export-blog-post-metrics.ts`:
17
+
18
+
```typescript
19
+
const MONITOR_ID = "1"; // The ID of the monitor to export
20
+
const PERIOD = "7d"; // Time period: "1d", "7d", or "14d"
21
+
const INTERVAL = 60; // Interval in minutes for data points
22
+
const TYPE = "http"; // Fallback monitor type: "http" or "tcp" (auto-detected from monitor)
23
+
const OUTPUT_FILE = "blog-post-metrics.json"; // Output filename
24
+
```
25
+
26
+
**Note:** The script automatically detects the monitor type from the database, but you can set a fallback with the `TYPE` constant.
27
+
28
+
## Prerequisites
29
+
30
+
1. Make sure you have the `TINY_BIRD_API_KEY` environment variable set in your `.env` file
31
+
2. The database should be accessible (local or remote)
32
+
3. Install dependencies: `pnpm install`
33
+
34
+
## Usage
35
+
36
+
> [!IMPORTANT]
37
+
> Go to the `/tinybird/src/client.ts` file and make sure tb is **not using the NoopClient**.
38
+
39
+
From the `apps/dashboard` directory:
40
+
41
+
```bash
42
+
# Using the npm script
43
+
pnpm export-metrics
44
+
45
+
# Or directly with bun
46
+
bun src/scripts/export-blog-post-metrics.ts
47
+
```
48
+
49
+
## Output Format
50
+
51
+
The script generates a JSON file with the following structure:
52
+
53
+
```json
54
+
{
55
+
"regions": ["ams", "fra", "lhr", ...],
56
+
"data": {
57
+
"regions": ["ams", "fra", "lhr", ...],
58
+
"data": [
59
+
{
60
+
"timestamp": "2025-08-18T16:00:00.000Z",
61
+
"ams": 207,
62
+
"fra": 142,
63
+
"lhr": 327,
64
+
...
65
+
}
66
+
]
67
+
},
68
+
"metricsByRegions": [
69
+
{
70
+
"region": "ams",
71
+
"count": 1000,
72
+
"ok": 995,
73
+
"p50Latency": 150,
74
+
"p75Latency": 200,
75
+
"p90Latency": 250,
76
+
"p95Latency": 300,
77
+
"p99Latency": 400
78
+
}
79
+
]
80
+
}
81
+
```
82
+
83
+
## Data Fields
84
+
85
+
- **regions**: Array of region codes and private location names for the monitor
86
+
- **data.data**: Timeline data with latency values per region/location at each timestamp
87
+
- **metricsByRegions**: Summary statistics per region/location including:
88
+
- `count`: Total number of checks
89
+
- `ok`: Number of successful checks
90
+
- `p50Latency`, `p75Latency`, `p90Latency`, `p95Latency`, `p99Latency`: Latency percentiles in milliseconds
91
+
92
+
**Note:** The script automatically includes both public Fly.io regions and any private locations connected to the monitor.
93
+
94
+
## Example: Moving to Web Assets
95
+
96
+
To use the exported data in the web app (like the existing `hono-cold.json`):
97
+
98
+
```bash
99
+
# After running the script
100
+
cp blog-post-metrics.json ../web/public/assets/posts/your-blog-post/data.json
101
+
```
102
+
103
+
## Troubleshooting
104
+
105
+
**Error: "TINY_BIRD_API_KEY environment variable is required"**
106
+
- Make sure you have the `TINY_BIRD_API_KEY` set in your `.env` file
107
+
108
+
**Error: "Monitor with ID X not found"**
109
+
- Verify the monitor ID exists in your database
110
+
- Check that you're connected to the correct database
111
+
112
+
**No data returned**
113
+
- Ensure the monitor has been running and collecting data for the specified period
114
+
- Try a different time period (e.g., "7d" instead of "1d")
115
+
+226
apps/dashboard/src/scripts/export-blog-post-metrics.ts
+226
apps/dashboard/src/scripts/export-blog-post-metrics.ts
···
1
+
import { writeFileSync } from "node:fs";
2
+
import { resolve } from "node:path";
3
+
import { db, eq } from "@openstatus/db";
4
+
import { monitor, selectMonitorSchema } from "@openstatus/db/src/schema";
5
+
import { OSTinybird } from "@openstatus/tinybird";
6
+
7
+
// WARNING: make sure to enable the Tinybird client in the env you are running this script in
8
+
9
+
// Configuration
10
+
const MONITOR_ID = "7002";
11
+
const PERIOD = "7d" as const;
12
+
const INTERVAL = 60;
13
+
const TYPE = "http" as const;
14
+
const OUTPUT_FILE = "blog-post-metrics.json";
15
+
const PERCENTILE = "p50"; // p50, p75, p90, p95, p99
16
+
17
+
async function main() {
18
+
// Get Tinybird API key from environment
19
+
const tinybirdApiKey = process.env.TINY_BIRD_API_KEY;
20
+
if (!tinybirdApiKey) {
21
+
throw new Error("TINY_BIRD_API_KEY environment variable is required");
22
+
}
23
+
24
+
const tb = new OSTinybird(tinybirdApiKey);
25
+
26
+
console.log(`Fetching data for monitor ID: ${MONITOR_ID}`);
27
+
28
+
// 1. Fetch monitor from database with private locations
29
+
const monitorDataRaw = await db.query.monitor.findFirst({
30
+
where: eq(monitor.id, Number.parseInt(MONITOR_ID)),
31
+
with: {
32
+
privateLocationToMonitors: {
33
+
with: {
34
+
privateLocation: true,
35
+
},
36
+
},
37
+
},
38
+
});
39
+
40
+
if (!monitorDataRaw) {
41
+
throw new Error(`Monitor with ID ${MONITOR_ID} not found`);
42
+
}
43
+
44
+
// Parse the monitor data using the schema to convert regions string to array
45
+
const monitorData = selectMonitorSchema.parse(monitorDataRaw);
46
+
47
+
// Get private location names
48
+
const privateLocationNames =
49
+
monitorDataRaw.privateLocationToMonitors
50
+
?.map((pl) => pl.privateLocation?.name)
51
+
.filter((name): name is string => Boolean(name)) || [];
52
+
53
+
// Combine regular regions with private locations
54
+
const allRegions = [...monitorData.regions, ...privateLocationNames];
55
+
56
+
console.log(`\nMonitor Details:`);
57
+
console.log(` ID: ${MONITOR_ID}`);
58
+
console.log(` Name: ${monitorData.name || "Unnamed"}`);
59
+
console.log(` Type: ${monitorData.jobType}`);
60
+
console.log(` Active: ${monitorData.active}`);
61
+
console.log(` Created: ${monitorData.createdAt}`);
62
+
console.log(` Regular regions: ${monitorData.regions.join(", ")}`);
63
+
console.log(
64
+
` Private locations: ${privateLocationNames.join(", ") || "None"}`
65
+
);
66
+
console.log(` Total regions: ${allRegions.length}`);
67
+
console.log(`\nQuery Parameters:`);
68
+
console.log(` Period: ${PERIOD}`);
69
+
console.log(` Interval: ${INTERVAL} minutes`);
70
+
71
+
// Use the monitor's actual type, or fall back to the configured TYPE
72
+
const monitorType = (monitorData.jobType || TYPE) as "http" | "tcp";
73
+
74
+
// 2. Fetch metricsRegions (timeline data with region, timestamp, and quantiles)
75
+
const metricsRegionsResult =
76
+
monitorType === "http"
77
+
? PERIOD === "7d"
78
+
? await tb.httpMetricsRegionsWeekly({
79
+
monitorId: MONITOR_ID,
80
+
interval: INTERVAL,
81
+
})
82
+
: await tb.httpMetricsRegionsDaily({
83
+
monitorId: MONITOR_ID,
84
+
interval: INTERVAL,
85
+
})
86
+
: PERIOD === "7d"
87
+
? await tb.tcpMetricsByIntervalWeekly({
88
+
monitorId: MONITOR_ID,
89
+
interval: INTERVAL,
90
+
})
91
+
: await tb.tcpMetricsByIntervalDaily({
92
+
monitorId: MONITOR_ID,
93
+
interval: INTERVAL,
94
+
});
95
+
96
+
console.log(
97
+
`\nFetched ${metricsRegionsResult.data.length} metrics regions data points`
98
+
);
99
+
if (metricsRegionsResult.data.length > 0) {
100
+
console.log(
101
+
` First data point:`,
102
+
JSON.stringify(metricsRegionsResult.data[0], null, 2)
103
+
);
104
+
console.log(
105
+
` Last data point:`,
106
+
JSON.stringify(
107
+
metricsRegionsResult.data[metricsRegionsResult.data.length - 1],
108
+
null,
109
+
2
110
+
)
111
+
);
112
+
} else {
113
+
console.log(` ⚠️ No data returned. This could mean:`);
114
+
console.log(` - The monitor hasn't collected any data yet`);
115
+
console.log(` - The monitor is inactive or was just created`);
116
+
console.log(
117
+
` - There's no data in the selected time period (${PERIOD})`
118
+
);
119
+
console.log(
120
+
`\n 💡 Tip: Try querying without the interval parameter or using PERIOD="1d"`
121
+
);
122
+
123
+
// Try without interval to see if that helps
124
+
console.log(`\n Trying without interval parameter...`);
125
+
const retryResult =
126
+
monitorType === "http"
127
+
? PERIOD === "7d"
128
+
? await tb.httpMetricsRegionsWeekly({
129
+
monitorId: MONITOR_ID,
130
+
})
131
+
: await tb.httpMetricsRegionsDaily({
132
+
monitorId: MONITOR_ID,
133
+
})
134
+
: PERIOD === "7d"
135
+
? await tb.tcpMetricsByIntervalWeekly({
136
+
monitorId: MONITOR_ID,
137
+
})
138
+
: await tb.tcpMetricsByIntervalDaily({
139
+
monitorId: MONITOR_ID,
140
+
});
141
+
console.log(` Retry returned ${retryResult.data.length} data points`);
142
+
if (retryResult.data.length > 0) {
143
+
console.log(` ✅ Success! The interval parameter might be the issue.`);
144
+
console.log(
145
+
` First data point:`,
146
+
JSON.stringify(retryResult.data[0], null, 2)
147
+
);
148
+
}
149
+
}
150
+
151
+
// 3. Fetch metricsByRegion (summary data by region)
152
+
const metricsByRegionProcedure =
153
+
monitorType === "http"
154
+
? PERIOD === "7d"
155
+
? tb.httpMetricsByRegionWeekly
156
+
: tb.httpMetricsByRegionDaily
157
+
: PERIOD === "7d"
158
+
? tb.tcpMetricsByRegionWeekly
159
+
: tb.tcpMetricsByRegionDaily;
160
+
161
+
const metricsByRegionsResult = await metricsByRegionProcedure({
162
+
monitorId: MONITOR_ID,
163
+
});
164
+
165
+
console.log(
166
+
`\nFetched ${metricsByRegionsResult.data.length} metrics by region data points`
167
+
);
168
+
if (metricsByRegionsResult.data.length > 0) {
169
+
console.log(
170
+
` Sample:`,
171
+
JSON.stringify(metricsByRegionsResult.data.slice(0, 3), null, 2)
172
+
);
173
+
}
174
+
175
+
// 4. Transform metricsRegions data to match expected format
176
+
// Group by timestamp and pivot regions as columns
177
+
const timelineMap = new Map<number, Record<string, number | string>>();
178
+
179
+
for (const row of metricsRegionsResult.data) {
180
+
const timestamp = row.timestamp;
181
+
const region = row.region;
182
+
const latency = row[`${PERCENTILE}Latency`] ?? 0;
183
+
184
+
if (!timelineMap.has(timestamp)) {
185
+
timelineMap.set(timestamp, {
186
+
timestamp: new Date(timestamp).toISOString(),
187
+
});
188
+
}
189
+
190
+
const entry = timelineMap.get(timestamp)!;
191
+
entry[region] = latency;
192
+
}
193
+
194
+
// Convert map to sorted array
195
+
const timelineData = Array.from(timelineMap.values()).sort((a, b) => {
196
+
const timeA = new Date(a.timestamp as string).getTime();
197
+
const timeB = new Date(b.timestamp as string).getTime();
198
+
return timeA - timeB;
199
+
});
200
+
201
+
// 5. Build final output structure
202
+
const output = {
203
+
regions: allRegions,
204
+
data: {
205
+
regions: allRegions,
206
+
data: timelineData,
207
+
},
208
+
metricsByRegions: metricsByRegionsResult.data,
209
+
};
210
+
211
+
// 6. Write to file
212
+
const outputPath = resolve(process.cwd(), OUTPUT_FILE);
213
+
writeFileSync(outputPath, JSON.stringify(output, null, 2));
214
+
215
+
console.log(`\n✅ Data exported successfully to: ${outputPath}`);
216
+
console.log(`Total timeline entries: ${timelineData.length}`);
217
+
console.log(
218
+
`Total regions (including private locations): ${allRegions.length}`
219
+
);
220
+
}
221
+
222
+
// Run the script
223
+
main().catch((error) => {
224
+
console.error("Error:", error);
225
+
process.exit(1);
226
+
});