atscan.net
wip
tree.fail
1package storage
2
3import (
4 "database/sql"
5 "fmt"
6 "path/filepath"
7 "time"
8)
9
10// DID represents a decentralized identifier
11type DID struct {
12 DID string
13 PDSEndpoint string
14 Handle string
15 CreatedAt time.Time
16 UpdatedAt time.Time
17}
18
19// Endpoint represents any AT Protocol service endpoint
20type Endpoint struct {
21 ID int64
22 EndpointType string
23 Endpoint string
24 ServerDID string
25 DiscoveredAt time.Time
26 LastChecked time.Time
27 Status int
28 IP string
29 IPv6 string
30 IPResolvedAt time.Time
31 Valid bool
32 UpdatedAt time.Time
33}
34
35// EndpointUpdate contains fields to update for an Endpoint
36type EndpointUpdate struct {
37 Status int
38 LastChecked time.Time
39 ResponseTime float64
40 ScanData *EndpointScanData
41}
42
43// EndpointScanData contains data from an endpoint scan
44type EndpointScanData struct {
45 ServerInfo interface{} `json:"server_info,omitempty"`
46 DIDs []string `json:"dids,omitempty"`
47 DIDCount int `json:"did_count"`
48 Metadata map[string]interface{} `json:"metadata,omitempty"`
49}
50
51// EndpointScan represents a historical endpoint scan
52type EndpointScan struct {
53 ID int64
54 EndpointID int64
55 Status int
56 ResponseTime float64
57 UserCount int64
58 Version string
59 UsedIP string // NEW: Track which IP was actually used
60 ScanData *EndpointScanData
61 ScannedAt time.Time
62}
63
64// Status constants
65const (
66 PDSStatusUnknown = 0
67 PDSStatusOnline = 1
68 PDSStatusOffline = 2
69)
70
71// Endpoint status constants (aliases for compatibility)
72const (
73 EndpointStatusUnknown = PDSStatusUnknown
74 EndpointStatusOnline = PDSStatusOnline
75 EndpointStatusOffline = PDSStatusOffline
76)
77
78// EndpointFilter for querying endpoints
79type EndpointFilter struct {
80 Type string
81 Status string
82 MinUserCount int64
83 OnlyStale bool
84 OnlyValid bool
85 RecheckInterval time.Duration
86 Random bool
87 Limit int
88 Offset int
89}
90
91// EndpointStats contains aggregate statistics about endpoints
92type EndpointStats struct {
93 TotalEndpoints int64 `json:"total_endpoints"`
94 ByType map[string]int64 `json:"by_type"`
95 OnlineEndpoints int64 `json:"online_endpoints"`
96 OfflineEndpoints int64 `json:"offline_endpoints"`
97 AvgResponseTime float64 `json:"avg_response_time"`
98 TotalDIDs int64 `json:"total_dids"` // Only for PDS
99}
100
101// Legacy type aliases for backward compatibility in code
102type PDS = Endpoint
103type PDSUpdate = EndpointUpdate
104type PDSScanData = EndpointScanData
105type PDSScan = EndpointScan
106type PDSFilter = EndpointFilter
107type PDSStats = EndpointStats
108
109// PLCMetrics contains metrics from PLC directory scans
110type PLCMetrics struct {
111 TotalDIDs int64 `json:"total_dids"`
112 TotalPDS int64 `json:"total_pds"`
113 UniquePDS int64 `json:"unique_pds"`
114 LastScanTime time.Time `json:"last_scan_time"`
115 ScanDuration int64 `json:"scan_duration_ms"`
116 ErrorCount int `json:"error_count"`
117}
118
119// PLCBundle represents a cached bundle of PLC operations
120type PLCBundle struct {
121 BundleNumber int
122 StartTime time.Time
123 EndTime time.Time
124 BoundaryCIDs []string
125 DIDCount int // Changed from DIDs []string
126 Hash string
127 CompressedHash string
128 CompressedSize int64
129 UncompressedSize int64
130 CumulativeCompressedSize int64
131 CumulativeUncompressedSize int64
132 Cursor string
133 PrevBundleHash string
134 Compressed bool
135 CreatedAt time.Time
136}
137
138// GetFilePath returns the computed file path for this bundle
139func (b *PLCBundle) GetFilePath(bundleDir string) string {
140 return filepath.Join(bundleDir, fmt.Sprintf("%06d.jsonl.zst", b.BundleNumber))
141}
142
143// OperationCount returns the number of operations in a bundle (always 10000)
144func (b *PLCBundle) OperationCount() int {
145 return 10000
146}
147
148type PLCHistoryPoint struct {
149 Date string `json:"date"`
150 BundleNumber int `json:"last_bundle_number"`
151 OperationCount int `json:"operations"`
152 UncompressedSize int64 `json:"size_uncompressed"`
153 CompressedSize int64 `json:"size_compressed"`
154 CumulativeUncompressed int64 `json:"cumulative_uncompressed"`
155 CumulativeCompressed int64 `json:"cumulative_compressed"`
156}
157
158// ScanCursor stores scanning progress
159type ScanCursor struct {
160 Source string
161 LastBundleNumber int
162 LastScanTime time.Time
163 RecordsProcessed int64
164}
165
166// DIDRecord represents a DID entry in the database
167type DIDRecord struct {
168 DID string `json:"did"`
169 Handle string `json:"handle,omitempty"`
170 CurrentPDS string `json:"current_pds,omitempty"`
171 LastOpAt time.Time `json:"last_op_at,omitempty"`
172 BundleNumbers []int `json:"bundle_numbers"`
173 CreatedAt time.Time `json:"created_at"`
174}
175
176// GlobalDIDInfo consolidates DID data from PLC and PDS tables
177type GlobalDIDInfo struct {
178 DIDRecord // Embeds all fields: DID, Handle, CurrentPDS, etc.
179 HostingOn []*PDSRepo `json:"hosting_on"`
180}
181
182// IPInfo represents IP information (stored with IP as primary key)
183type IPInfo struct {
184 IP string `json:"ip"`
185 City string `json:"city,omitempty"`
186 Country string `json:"country,omitempty"`
187 CountryCode string `json:"country_code,omitempty"`
188 ASN int `json:"asn,omitempty"`
189 ASNOrg string `json:"asn_org,omitempty"`
190 IsDatacenter bool `json:"is_datacenter"`
191 IsVPN bool `json:"is_vpn"`
192 IsCrawler bool `json:"is_crawler"`
193 IsTor bool `json:"is_tor"`
194 IsProxy bool `json:"is_proxy"`
195 Latitude float32 `json:"latitude,omitempty"`
196 Longitude float32 `json:"longitude,omitempty"`
197 RawData map[string]interface{} `json:"raw_data,omitempty"`
198 FetchedAt time.Time `json:"fetched_at"`
199 UpdatedAt time.Time `json:"updated_at"`
200}
201
202// IsHome returns true if this is a residential/home IP
203// (not crawler, datacenter, tor, proxy, or vpn)
204func (i *IPInfo) IsHome() bool {
205 return !i.IsCrawler && !i.IsDatacenter && !i.IsTor && !i.IsProxy && !i.IsVPN
206}
207
208// PDSListItem is a virtual type created by JOIN for /pds endpoint
209type PDSListItem struct {
210 // From endpoints table
211 ID int64
212 Endpoint string
213 ServerDID string
214 DiscoveredAt time.Time
215 LastChecked time.Time
216 Status int
217 IP string
218 IPv6 string
219 Valid bool // NEW
220
221 // From latest endpoint_scans (via JOIN)
222 LatestScan *struct {
223 UserCount int
224 ResponseTime float64
225 Version string
226 ScannedAt time.Time
227 }
228
229 // From ip_infos table (via JOIN on endpoints.ip)
230 IPInfo *IPInfo
231}
232
233// PDSDetail is extended version for /pds/{endpoint}
234type PDSDetail struct {
235 PDSListItem
236
237 // Additional data from latest scan
238 LatestScan *struct {
239 UserCount int
240 ResponseTime float64
241 Version string
242 ServerInfo interface{} // Full server description
243 ScannedAt time.Time
244 }
245
246 // NEW: Aliases (other domains pointing to same server)
247 Aliases []string `json:"aliases,omitempty"`
248 IsPrimary bool `json:"is_primary"`
249}
250
251type CountryStats struct {
252 Country string `json:"country"`
253 CountryCode string `json:"country_code"`
254 ActivePDSCount int64 `json:"active_pds_count"`
255 PDSPercentage float64 `json:"pds_percentage"`
256 TotalUsers int64 `json:"total_users"`
257 UsersPercentage float64 `json:"users_percentage"`
258 AvgResponseTimeMS float64 `json:"avg_response_time_ms"`
259}
260
261type VersionStats struct {
262 Version string `json:"version"`
263 PDSCount int64 `json:"pds_count"`
264 Percentage float64 `json:"percentage"`
265 PercentageText string `json:"percentage_text"`
266 TotalUsers int64 `json:"total_users"`
267 UsersPercentage float64 `json:"users_percentage"`
268 FirstSeen time.Time `json:"first_seen"`
269 LastSeen time.Time `json:"last_seen"`
270}
271
272type PDSRepo struct {
273 ID int64 `json:"id"`
274 EndpointID int64 `json:"endpoint_id"`
275 Endpoint string `json:"endpoint,omitempty"`
276 DID string `json:"did"`
277 Head string `json:"head,omitempty"`
278 Rev string `json:"rev,omitempty"`
279 Active bool `json:"active"`
280 Status string `json:"status,omitempty"`
281 FirstSeen time.Time `json:"first_seen"`
282 LastSeen time.Time `json:"last_seen"`
283 UpdatedAt time.Time `json:"updated_at"`
284}
285
286type PDSRepoData struct {
287 DID string
288 Head string
289 Rev string
290 Active bool
291 Status string
292}
293
294type DIDBackfillInfo struct {
295 DID string
296 LastBundleNum int
297}
298
299type DIDStateUpdateData struct {
300 DID string
301 Handle sql.NullString // Use sql.NullString for potential NULLs
302 PDS sql.NullString
303 OpTime time.Time
304}
305
306// TableSizeInfo holds size information for a database table.
307type TableSizeInfo struct {
308 TableName string `json:"table_name"`
309 TotalBytes int64 `json:"total_bytes"` // Raw bytes
310 TableHeapBytes int64 `json:"table_heap_bytes"` // Raw bytes
311 IndexesBytes int64 `json:"indexes_bytes"` // Raw bytes
312}
313
314// IndexSizeInfo holds size information for a database index.
315type IndexSizeInfo struct {
316 IndexName string `json:"index_name"`
317 TableName string `json:"table_name"`
318 IndexBytes int64 `json:"index_bytes"` // Raw bytes
319}