+51

objectstore/mix/bytes.go

··· 1 1 + package mix 2 2 + 3 3 + import ( 4 4 + "errors" 5 5 + "fmt" 6 6 + 7 7 + "codeberg.org/lindenii/furgit/objectid" 8 8 + "codeberg.org/lindenii/furgit/objectstore" 9 9 + "codeberg.org/lindenii/furgit/objecttype" 10 10 + ) 11 11 + 12 12 + // ReadBytesFull reads a full serialized object from one backend that has it. 13 13 + func (mix *Mix) ReadBytesFull(id objectid.ObjectID) ([]byte, error) { 14 14 + for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 15 15 + full, err := backend.ReadBytesFull(id) 16 16 + if err == nil { 17 17 + mix.touchBackend(backend) 18 18 + 19 19 + return full, nil 20 20 + } 21 21 + 22 22 + if errors.Is(err, objectstore.ErrObjectNotFound) { 23 23 + continue 24 24 + } 25 25 + 26 26 + return nil, fmt.Errorf("objectstore: backend %d read bytes full: %w", i, err) 27 27 + } 28 28 + 29 29 + return nil, objectstore.ErrObjectNotFound 30 30 + } 31 31 + 32 32 + // ReadBytesContent reads an object's type and content bytes from one backend 33 33 + // that has it. 34 34 + func (mix *Mix) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []byte, error) { 35 35 + for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 36 36 + ty, content, err := backend.ReadBytesContent(id) 37 37 + if err == nil { 38 38 + mix.touchBackend(backend) 39 39 + 40 40 + return ty, content, nil 41 41 + } 42 42 + 43 43 + if errors.Is(err, objectstore.ErrObjectNotFound) { 44 44 + continue 45 45 + } 46 46 + 47 47 + return objecttype.TypeInvalid, nil, fmt.Errorf("objectstore: backend %d read bytes content: %w", i, err) 48 48 + } 49 49 + 50 50 + return objecttype.TypeInvalid, nil, objectstore.ErrObjectNotFound 51 51 + }

+30

objectstore/mix/close.go

··· 1 1 + package mix 2 2 + 3 3 + import ( 4 4 + "errors" 5 5 + 6 6 + "codeberg.org/lindenii/furgit/objectstore" 7 7 + ) 8 8 + 9 9 + // Close closes all backends and joins close errors. 10 10 + func (mix *Mix) Close() error { 11 11 + mix.mu.RLock() 12 12 + 13 13 + backends := make([]objectstore.Store, 0, len(mix.backendNodeByStore)) 14 14 + for node := mix.backendHead; node != nil; node = node.next { 15 15 + backends = append(backends, node.backend) 16 16 + } 17 17 + 18 18 + mix.mu.RUnlock() 19 19 + 20 20 + var errs []error 21 21 + 22 22 + for _, backend := range backends { 23 23 + err := backend.Close() 24 24 + if err != nil { 25 25 + errs = append(errs, err) 26 26 + } 27 27 + } 28 28 + 29 29 + return errors.Join(errs...) 30 30 + }

+30

objectstore/mix/header.go

··· 1 1 + package mix 2 2 + 3 3 + import ( 4 4 + "errors" 5 5 + "fmt" 6 6 + 7 7 + "codeberg.org/lindenii/furgit/objectid" 8 8 + "codeberg.org/lindenii/furgit/objectstore" 9 9 + "codeberg.org/lindenii/furgit/objecttype" 10 10 + ) 11 11 + 12 12 + // ReadHeader reads object header data from one backend that has it. 13 13 + func (mix *Mix) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, error) { 14 14 + for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 15 15 + ty, size, err := backend.ReadHeader(id) 16 16 + if err == nil { 17 17 + mix.touchBackend(backend) 18 18 + 19 19 + return ty, size, nil 20 20 + } 21 21 + 22 22 + if errors.Is(err, objectstore.ErrObjectNotFound) { 23 23 + continue 24 24 + } 25 25 + 26 26 + return objecttype.TypeInvalid, 0, fmt.Errorf("objectstore: backend %d read header: %w", i, err) 27 27 + } 28 28 + 29 29 + return objecttype.TypeInvalid, 0, objectstore.ErrObjectNotFound 30 30 + }

-257

objectstore/mix/mix.go

··· 3 3 package mix 4 4 5 5 import ( 6 6 - "errors" 7 7 - "fmt" 8 8 - "io" 9 6 "sync" 10 7 11 11 - "codeberg.org/lindenii/furgit/objectid" 12 8 "codeberg.org/lindenii/furgit/objectstore" 13 13 - "codeberg.org/lindenii/furgit/objecttype" 14 9 ) 15 10 16 11 // Mix queries multiple object databases with an MRU backend preference. ··· 21 16 backendTail *backendNode 22 17 backendNodeByStore map[objectstore.Store]*backendNode 23 18 } 24 24 - 25 25 - // New creates a Mix from backends. 26 26 - func New(backends ...objectstore.Store) *Mix { 27 27 - nodeByStore := make(map[objectstore.Store]*backendNode, len(backends)) 28 28 - 29 29 - var ( 30 30 - head *backendNode 31 31 - tail *backendNode 32 32 - ) 33 33 - 34 34 - for _, backend := range backends { 35 35 - if backend == nil { 36 36 - continue 37 37 - } 38 38 - 39 39 - node := &backendNode{ 40 40 - backend: backend, 41 41 - prev: tail, 42 42 - } 43 43 - if tail != nil { 44 44 - tail.next = node 45 45 - } 46 46 - 47 47 - if head == nil { 48 48 - head = node 49 49 - } 50 50 - 51 51 - tail = node 52 52 - nodeByStore[backend] = node 53 53 - } 54 54 - 55 55 - return &Mix{ 56 56 - backendHead: head, 57 57 - backendTail: tail, 58 58 - backendNodeByStore: nodeByStore, 59 59 - } 60 60 - } 61 61 - 62 62 - // ReadBytesFull reads a full serialized object from one backend that has it. 63 63 - func (mix *Mix) ReadBytesFull(id objectid.ObjectID) ([]byte, error) { 64 64 - for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 65 65 - full, err := backend.ReadBytesFull(id) 66 66 - if err == nil { 67 67 - mix.touchBackend(backend) 68 68 - 69 69 - return full, nil 70 70 - } 71 71 - 72 72 - if errors.Is(err, objectstore.ErrObjectNotFound) { 73 73 - continue 74 74 - } 75 75 - 76 76 - return nil, fmt.Errorf("objectstore: backend %d read bytes full: %w", i, err) 77 77 - } 78 78 - 79 79 - return nil, objectstore.ErrObjectNotFound 80 80 - } 81 81 - 82 82 - // ReadBytesContent reads an object's type and content bytes from one backend 83 83 - // that has it. 84 84 - func (mix *Mix) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []byte, error) { 85 85 - for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 86 86 - ty, content, err := backend.ReadBytesContent(id) 87 87 - if err == nil { 88 88 - mix.touchBackend(backend) 89 89 - 90 90 - return ty, content, nil 91 91 - } 92 92 - 93 93 - if errors.Is(err, objectstore.ErrObjectNotFound) { 94 94 - continue 95 95 - } 96 96 - 97 97 - return objecttype.TypeInvalid, nil, fmt.Errorf("objectstore: backend %d read bytes content: %w", i, err) 98 98 - } 99 99 - 100 100 - return objecttype.TypeInvalid, nil, objectstore.ErrObjectNotFound 101 101 - } 102 102 - 103 103 - // ReadReaderFull reads a full serialized object stream from one backend that 104 104 - // has it. 105 105 - func (mix *Mix) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) { 106 106 - for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 107 107 - reader, err := backend.ReadReaderFull(id) 108 108 - if err == nil { 109 109 - mix.touchBackend(backend) 110 110 - 111 111 - return reader, nil 112 112 - } 113 113 - 114 114 - if errors.Is(err, objectstore.ErrObjectNotFound) { 115 115 - continue 116 116 - } 117 117 - 118 118 - return nil, fmt.Errorf("objectstore: backend %d read reader full: %w", i, err) 119 119 - } 120 120 - 121 121 - return nil, objectstore.ErrObjectNotFound 122 122 - } 123 123 - 124 124 - // ReadReaderContent reads an object's type, declared content length, and 125 125 - // content stream from one backend that has it. 126 126 - func (mix *Mix) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, int64, io.ReadCloser, error) { 127 127 - for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 128 128 - ty, size, reader, err := backend.ReadReaderContent(id) 129 129 - if err == nil { 130 130 - mix.touchBackend(backend) 131 131 - 132 132 - return ty, size, reader, nil 133 133 - } 134 134 - 135 135 - if errors.Is(err, objectstore.ErrObjectNotFound) { 136 136 - continue 137 137 - } 138 138 - 139 139 - return objecttype.TypeInvalid, 0, nil, fmt.Errorf("objectstore: backend %d read reader content: %w", i, err) 140 140 - } 141 141 - 142 142 - return objecttype.TypeInvalid, 0, nil, objectstore.ErrObjectNotFound 143 143 - } 144 144 - 145 145 - // ReadSize reads object content length from one backend that has it. 146 146 - func (mix *Mix) ReadSize(id objectid.ObjectID) (int64, error) { 147 147 - for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 148 148 - size, err := backend.ReadSize(id) 149 149 - if err == nil { 150 150 - mix.touchBackend(backend) 151 151 - 152 152 - return size, nil 153 153 - } 154 154 - 155 155 - if errors.Is(err, objectstore.ErrObjectNotFound) { 156 156 - continue 157 157 - } 158 158 - 159 159 - return 0, fmt.Errorf("objectstore: backend %d read size: %w", i, err) 160 160 - } 161 161 - 162 162 - return 0, objectstore.ErrObjectNotFound 163 163 - } 164 164 - 165 165 - // ReadHeader reads object header data from one backend that has it. 166 166 - func (mix *Mix) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, error) { 167 167 - for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 168 168 - ty, size, err := backend.ReadHeader(id) 169 169 - if err == nil { 170 170 - mix.touchBackend(backend) 171 171 - 172 172 - return ty, size, nil 173 173 - } 174 174 - 175 175 - if errors.Is(err, objectstore.ErrObjectNotFound) { 176 176 - continue 177 177 - } 178 178 - 179 179 - return objecttype.TypeInvalid, 0, fmt.Errorf("objectstore: backend %d read header: %w", i, err) 180 180 - } 181 181 - 182 182 - return objecttype.TypeInvalid, 0, objectstore.ErrObjectNotFound 183 183 - } 184 184 - 185 185 - // Close closes all backends and joins close errors. 186 186 - func (mix *Mix) Close() error { 187 187 - mix.mu.RLock() 188 188 - 189 189 - backends := make([]objectstore.Store, 0, len(mix.backendNodeByStore)) 190 190 - for node := mix.backendHead; node != nil; node = node.next { 191 191 - backends = append(backends, node.backend) 192 192 - } 193 193 - 194 194 - mix.mu.RUnlock() 195 195 - 196 196 - var errs []error 197 197 - 198 198 - for _, backend := range backends { 199 199 - err := backend.Close() 200 200 - if err != nil { 201 201 - errs = append(errs, err) 202 202 - } 203 203 - } 204 204 - 205 205 - return errors.Join(errs...) 206 206 - } 207 207 - 208 208 - type backendNode struct { 209 209 - backend objectstore.Store 210 210 - prev *backendNode 211 211 - next *backendNode 212 212 - } 213 213 - 214 214 - func (mix *Mix) firstBackend() objectstore.Store { 215 215 - mix.mu.RLock() 216 216 - defer mix.mu.RUnlock() 217 217 - 218 218 - if mix.backendHead == nil { 219 219 - return nil 220 220 - } 221 221 - 222 222 - return mix.backendHead.backend 223 223 - } 224 224 - 225 225 - func (mix *Mix) nextBackend(current objectstore.Store) objectstore.Store { 226 226 - mix.mu.RLock() 227 227 - defer mix.mu.RUnlock() 228 228 - 229 229 - node := mix.backendNodeByStore[current] 230 230 - if node == nil || node.next == nil { 231 231 - return nil 232 232 - } 233 233 - 234 234 - return node.next.backend 235 235 - } 236 236 - 237 237 - func (mix *Mix) touchBackend(backend objectstore.Store) { 238 238 - if backend == nil { 239 239 - return 240 240 - } 241 241 - 242 242 - if !mix.mu.TryLock() { 243 243 - return 244 244 - } 245 245 - defer mix.mu.Unlock() 246 246 - 247 247 - node := mix.backendNodeByStore[backend] 248 248 - if node == nil || node == mix.backendHead { 249 249 - return 250 250 - } 251 251 - 252 252 - if node.prev != nil { 253 253 - node.prev.next = node.next 254 254 - } 255 255 - 256 256 - if node.next != nil { 257 257 - node.next.prev = node.prev 258 258 - } 259 259 - 260 260 - if mix.backendTail == node { 261 261 - mix.backendTail = node.prev 262 262 - } 263 263 - 264 264 - node.prev = nil 265 265 - 266 266 - node.next = mix.backendHead 267 267 - if mix.backendHead != nil { 268 268 - mix.backendHead.prev = node 269 269 - } 270 270 - 271 271 - mix.backendHead = node 272 272 - if mix.backendTail == nil { 273 273 - mix.backendTail = node 274 274 - } 275 275 - }

+72

objectstore/mix/mru.go

··· 1 1 + package mix 2 2 + 3 3 + import "codeberg.org/lindenii/furgit/objectstore" 4 4 + 5 5 + type backendNode struct { 6 6 + backend objectstore.Store 7 7 + prev *backendNode 8 8 + next *backendNode 9 9 + } 10 10 + 11 11 + func (mix *Mix) firstBackend() objectstore.Store { 12 12 + mix.mu.RLock() 13 13 + defer mix.mu.RUnlock() 14 14 + 15 15 + if mix.backendHead == nil { 16 16 + return nil 17 17 + } 18 18 + 19 19 + return mix.backendHead.backend 20 20 + } 21 21 + 22 22 + func (mix *Mix) nextBackend(current objectstore.Store) objectstore.Store { 23 23 + mix.mu.RLock() 24 24 + defer mix.mu.RUnlock() 25 25 + 26 26 + node := mix.backendNodeByStore[current] 27 27 + if node == nil || node.next == nil { 28 28 + return nil 29 29 + } 30 30 + 31 31 + return node.next.backend 32 32 + } 33 33 + 34 34 + func (mix *Mix) touchBackend(backend objectstore.Store) { 35 35 + if backend == nil { 36 36 + return 37 37 + } 38 38 + 39 39 + if !mix.mu.TryLock() { 40 40 + return 41 41 + } 42 42 + defer mix.mu.Unlock() 43 43 + 44 44 + node := mix.backendNodeByStore[backend] 45 45 + if node == nil || node == mix.backendHead { 46 46 + return 47 47 + } 48 48 + 49 49 + if node.prev != nil { 50 50 + node.prev.next = node.next 51 51 + } 52 52 + 53 53 + if node.next != nil { 54 54 + node.next.prev = node.prev 55 55 + } 56 56 + 57 57 + if mix.backendTail == node { 58 58 + mix.backendTail = node.prev 59 59 + } 60 60 + 61 61 + node.prev = nil 62 62 + 63 63 + node.next = mix.backendHead 64 64 + if mix.backendHead != nil { 65 65 + mix.backendHead.prev = node 66 66 + } 67 67 + 68 68 + mix.backendHead = node 69 69 + if mix.backendTail == nil { 70 70 + mix.backendTail = node 71 71 + } 72 72 + }

+40

objectstore/mix/new.go

··· 1 1 + package mix 2 2 + 3 3 + import "codeberg.org/lindenii/furgit/objectstore" 4 4 + 5 5 + // New creates a Mix from backends. 6 6 + func New(backends ...objectstore.Store) *Mix { 7 7 + nodeByStore := make(map[objectstore.Store]*backendNode, len(backends)) 8 8 + 9 9 + var ( 10 10 + head *backendNode 11 11 + tail *backendNode 12 12 + ) 13 13 + 14 14 + for _, backend := range backends { 15 15 + if backend == nil { 16 16 + continue 17 17 + } 18 18 + 19 19 + node := &backendNode{ 20 20 + backend: backend, 21 21 + prev: tail, 22 22 + } 23 23 + if tail != nil { 24 24 + tail.next = node 25 25 + } 26 26 + 27 27 + if head == nil { 28 28 + head = node 29 29 + } 30 30 + 31 31 + tail = node 32 32 + nodeByStore[backend] = node 33 33 + } 34 34 + 35 35 + return &Mix{ 36 36 + backendHead: head, 37 37 + backendTail: tail, 38 38 + backendNodeByStore: nodeByStore, 39 39 + } 40 40 + }

+53

objectstore/mix/reader.go

··· 1 1 + package mix 2 2 + 3 3 + import ( 4 4 + "errors" 5 5 + "fmt" 6 6 + "io" 7 7 + 8 8 + "codeberg.org/lindenii/furgit/objectid" 9 9 + "codeberg.org/lindenii/furgit/objectstore" 10 10 + "codeberg.org/lindenii/furgit/objecttype" 11 11 + ) 12 12 + 13 13 + // ReadReaderFull reads a full serialized object stream from one backend that 14 14 + // has it. 15 15 + func (mix *Mix) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) { 16 16 + for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 17 17 + reader, err := backend.ReadReaderFull(id) 18 18 + if err == nil { 19 19 + mix.touchBackend(backend) 20 20 + 21 21 + return reader, nil 22 22 + } 23 23 + 24 24 + if errors.Is(err, objectstore.ErrObjectNotFound) { 25 25 + continue 26 26 + } 27 27 + 28 28 + return nil, fmt.Errorf("objectstore: backend %d read reader full: %w", i, err) 29 29 + } 30 30 + 31 31 + return nil, objectstore.ErrObjectNotFound 32 32 + } 33 33 + 34 34 + // ReadReaderContent reads an object's type, declared content length, and 35 35 + // content stream from one backend that has it. 36 36 + func (mix *Mix) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, int64, io.ReadCloser, error) { 37 37 + for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 38 38 + ty, size, reader, err := backend.ReadReaderContent(id) 39 39 + if err == nil { 40 40 + mix.touchBackend(backend) 41 41 + 42 42 + return ty, size, reader, nil 43 43 + } 44 44 + 45 45 + if errors.Is(err, objectstore.ErrObjectNotFound) { 46 46 + continue 47 47 + } 48 48 + 49 49 + return objecttype.TypeInvalid, 0, nil, fmt.Errorf("objectstore: backend %d read reader content: %w", i, err) 50 50 + } 51 51 + 52 52 + return objecttype.TypeInvalid, 0, nil, objectstore.ErrObjectNotFound 53 53 + }

+29

objectstore/mix/size.go

··· 1 1 + package mix 2 2 + 3 3 + import ( 4 4 + "errors" 5 5 + "fmt" 6 6 + 7 7 + "codeberg.org/lindenii/furgit/objectid" 8 8 + "codeberg.org/lindenii/furgit/objectstore" 9 9 + ) 10 10 + 11 11 + // ReadSize reads object content length from one backend that has it. 12 12 + func (mix *Mix) ReadSize(id objectid.ObjectID) (int64, error) { 13 13 + for i, backend := 0, mix.firstBackend(); backend != nil; i, backend = i+1, mix.nextBackend(backend) { 14 14 + size, err := backend.ReadSize(id) 15 15 + if err == nil { 16 16 + mix.touchBackend(backend) 17 17 + 18 18 + return size, nil 19 19 + } 20 20 + 21 21 + if errors.Is(err, objectstore.ErrObjectNotFound) { 22 22 + continue 23 23 + } 24 24 + 25 25 + return 0, fmt.Errorf("objectstore: backend %d read size: %w", i, err) 26 26 + } 27 27 + 28 28 + return 0, objectstore.ErrObjectNotFound 29 29 + }