ilyagr.bsky.social
just playing with tangled
1// Copyright 2020 The Jujutsu Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(missing_docs)]
16
17use std::any::Any;
18use std::fmt::Debug;
19use std::fs;
20use std::fs::File;
21use std::io::Read;
22use std::io::Write as _;
23use std::path::Path;
24use std::path::PathBuf;
25use std::time::SystemTime;
26
27use async_trait::async_trait;
28use blake2::Blake2b512;
29use blake2::Digest as _;
30use futures::stream;
31use futures::stream::BoxStream;
32use pollster::FutureExt as _;
33use prost::Message as _;
34use tempfile::NamedTempFile;
35
36use crate::backend::make_root_commit;
37use crate::backend::Backend;
38use crate::backend::BackendError;
39use crate::backend::BackendResult;
40use crate::backend::ChangeId;
41use crate::backend::Commit;
42use crate::backend::CommitId;
43use crate::backend::Conflict;
44use crate::backend::ConflictId;
45use crate::backend::ConflictTerm;
46use crate::backend::CopyRecord;
47use crate::backend::FileId;
48use crate::backend::MergedTreeId;
49use crate::backend::MillisSinceEpoch;
50use crate::backend::SecureSig;
51use crate::backend::Signature;
52use crate::backend::SigningFn;
53use crate::backend::SymlinkId;
54use crate::backend::Timestamp;
55use crate::backend::Tree;
56use crate::backend::TreeId;
57use crate::backend::TreeValue;
58use crate::content_hash::blake2b_hash;
59use crate::file_util::persist_content_addressed_temp_file;
60use crate::index::Index;
61use crate::merge::MergeBuilder;
62use crate::object_id::ObjectId;
63use crate::repo_path::RepoPath;
64use crate::repo_path::RepoPathBuf;
65use crate::repo_path::RepoPathComponentBuf;
66
67const COMMIT_ID_LENGTH: usize = 64;
68const CHANGE_ID_LENGTH: usize = 16;
69
70fn map_not_found_err(err: std::io::Error, id: &impl ObjectId) -> BackendError {
71 if err.kind() == std::io::ErrorKind::NotFound {
72 BackendError::ObjectNotFound {
73 object_type: id.object_type(),
74 hash: id.hex(),
75 source: Box::new(err),
76 }
77 } else {
78 BackendError::ReadObject {
79 object_type: id.object_type(),
80 hash: id.hex(),
81 source: Box::new(err),
82 }
83 }
84}
85
86fn to_other_err(err: impl Into<Box<dyn std::error::Error + Send + Sync>>) -> BackendError {
87 BackendError::Other(err.into())
88}
89
90#[derive(Debug)]
91pub struct SimpleBackend {
92 path: PathBuf,
93 root_commit_id: CommitId,
94 root_change_id: ChangeId,
95 empty_tree_id: TreeId,
96}
97
98impl SimpleBackend {
99 pub fn name() -> &'static str {
100 "Simple"
101 }
102
103 pub fn init(store_path: &Path) -> Self {
104 fs::create_dir(store_path.join("commits")).unwrap();
105 fs::create_dir(store_path.join("trees")).unwrap();
106 fs::create_dir(store_path.join("files")).unwrap();
107 fs::create_dir(store_path.join("symlinks")).unwrap();
108 fs::create_dir(store_path.join("conflicts")).unwrap();
109 let backend = Self::load(store_path);
110 let empty_tree_id = backend
111 .write_tree(RepoPath::root(), &Tree::default())
112 .block_on()
113 .unwrap();
114 assert_eq!(empty_tree_id, backend.empty_tree_id);
115 backend
116 }
117
118 pub fn load(store_path: &Path) -> Self {
119 let root_commit_id = CommitId::from_bytes(&[0; COMMIT_ID_LENGTH]);
120 let root_change_id = ChangeId::from_bytes(&[0; CHANGE_ID_LENGTH]);
121 let empty_tree_id = TreeId::from_hex(
122 "482ae5a29fbe856c7272f2071b8b0f0359ee2d89ff392b8a900643fbd0836eccd067b8bf41909e206c90d45d6e7d8b6686b93ecaee5fe1a9060d87b672101310",
123 );
124 SimpleBackend {
125 path: store_path.to_path_buf(),
126 root_commit_id,
127 root_change_id,
128 empty_tree_id,
129 }
130 }
131
132 fn file_path(&self, id: &FileId) -> PathBuf {
133 self.path.join("files").join(id.hex())
134 }
135
136 fn symlink_path(&self, id: &SymlinkId) -> PathBuf {
137 self.path.join("symlinks").join(id.hex())
138 }
139
140 fn tree_path(&self, id: &TreeId) -> PathBuf {
141 self.path.join("trees").join(id.hex())
142 }
143
144 fn commit_path(&self, id: &CommitId) -> PathBuf {
145 self.path.join("commits").join(id.hex())
146 }
147
148 fn conflict_path(&self, id: &ConflictId) -> PathBuf {
149 self.path.join("conflicts").join(id.hex())
150 }
151}
152
153#[async_trait]
154impl Backend for SimpleBackend {
155 fn as_any(&self) -> &dyn Any {
156 self
157 }
158
159 fn name(&self) -> &str {
160 Self::name()
161 }
162
163 fn commit_id_length(&self) -> usize {
164 COMMIT_ID_LENGTH
165 }
166
167 fn change_id_length(&self) -> usize {
168 CHANGE_ID_LENGTH
169 }
170
171 fn root_commit_id(&self) -> &CommitId {
172 &self.root_commit_id
173 }
174
175 fn root_change_id(&self) -> &ChangeId {
176 &self.root_change_id
177 }
178
179 fn empty_tree_id(&self) -> &TreeId {
180 &self.empty_tree_id
181 }
182
183 fn concurrency(&self) -> usize {
184 1
185 }
186
187 async fn read_file(&self, _path: &RepoPath, id: &FileId) -> BackendResult<Box<dyn Read>> {
188 let path = self.file_path(id);
189 let file = File::open(path).map_err(|err| map_not_found_err(err, id))?;
190 Ok(Box::new(file))
191 }
192
193 async fn write_file(
194 &self,
195 _path: &RepoPath,
196 contents: &mut (dyn Read + Send),
197 ) -> BackendResult<FileId> {
198 // TODO: Write temporary file in the destination directory (#5712)
199 let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
200 let mut file = temp_file.as_file();
201 let mut hasher = Blake2b512::new();
202 let mut buff: Vec<u8> = vec![0; 1 << 14];
203 loop {
204 let bytes_read = contents.read(&mut buff).map_err(to_other_err)?;
205 if bytes_read == 0 {
206 break;
207 }
208 let bytes = &buff[..bytes_read];
209 file.write_all(bytes).map_err(to_other_err)?;
210 hasher.update(bytes);
211 }
212 file.flush().map_err(to_other_err)?;
213 let id = FileId::new(hasher.finalize().to_vec());
214
215 persist_content_addressed_temp_file(temp_file, self.file_path(&id))
216 .map_err(to_other_err)?;
217 Ok(id)
218 }
219
220 async fn read_symlink(&self, _path: &RepoPath, id: &SymlinkId) -> BackendResult<String> {
221 let path = self.symlink_path(id);
222 let target = fs::read_to_string(path).map_err(|err| map_not_found_err(err, id))?;
223 Ok(target)
224 }
225
226 async fn write_symlink(&self, _path: &RepoPath, target: &str) -> BackendResult<SymlinkId> {
227 // TODO: Write temporary file in the destination directory (#5712)
228 let mut temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
229 temp_file
230 .write_all(target.as_bytes())
231 .map_err(to_other_err)?;
232 let mut hasher = Blake2b512::new();
233 hasher.update(target.as_bytes());
234 let id = SymlinkId::new(hasher.finalize().to_vec());
235
236 persist_content_addressed_temp_file(temp_file, self.symlink_path(&id))
237 .map_err(to_other_err)?;
238 Ok(id)
239 }
240
241 async fn read_tree(&self, _path: &RepoPath, id: &TreeId) -> BackendResult<Tree> {
242 let path = self.tree_path(id);
243 let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
244
245 let proto = crate::protos::simple_store::Tree::decode(&*buf).map_err(to_other_err)?;
246 Ok(tree_from_proto(proto))
247 }
248
249 async fn write_tree(&self, _path: &RepoPath, tree: &Tree) -> BackendResult<TreeId> {
250 // TODO: Write temporary file in the destination directory (#5712)
251 let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
252
253 let proto = tree_to_proto(tree);
254 temp_file
255 .as_file()
256 .write_all(&proto.encode_to_vec())
257 .map_err(to_other_err)?;
258
259 let id = TreeId::new(blake2b_hash(tree).to_vec());
260
261 persist_content_addressed_temp_file(temp_file, self.tree_path(&id))
262 .map_err(to_other_err)?;
263 Ok(id)
264 }
265
266 fn read_conflict(&self, _path: &RepoPath, id: &ConflictId) -> BackendResult<Conflict> {
267 let path = self.conflict_path(id);
268 let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
269
270 let proto = crate::protos::simple_store::Conflict::decode(&*buf).map_err(to_other_err)?;
271 Ok(conflict_from_proto(proto))
272 }
273
274 fn write_conflict(&self, _path: &RepoPath, conflict: &Conflict) -> BackendResult<ConflictId> {
275 // TODO: Write temporary file in the destination directory (#5712)
276 let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
277
278 let proto = conflict_to_proto(conflict);
279 temp_file
280 .as_file()
281 .write_all(&proto.encode_to_vec())
282 .map_err(to_other_err)?;
283
284 let id = ConflictId::new(blake2b_hash(conflict).to_vec());
285
286 persist_content_addressed_temp_file(temp_file, self.conflict_path(&id))
287 .map_err(to_other_err)?;
288 Ok(id)
289 }
290
291 async fn read_commit(&self, id: &CommitId) -> BackendResult<Commit> {
292 if *id == self.root_commit_id {
293 return Ok(make_root_commit(
294 self.root_change_id().clone(),
295 self.empty_tree_id.clone(),
296 ));
297 }
298
299 let path = self.commit_path(id);
300 let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
301
302 let proto = crate::protos::simple_store::Commit::decode(&*buf).map_err(to_other_err)?;
303 Ok(commit_from_proto(proto))
304 }
305
306 async fn write_commit(
307 &self,
308 mut commit: Commit,
309 sign_with: Option<&mut SigningFn>,
310 ) -> BackendResult<(CommitId, Commit)> {
311 assert!(commit.secure_sig.is_none(), "commit.secure_sig was set");
312
313 if commit.parents.is_empty() {
314 return Err(BackendError::Other(
315 "Cannot write a commit with no parents".into(),
316 ));
317 }
318 // TODO: Write temporary file in the destination directory (#5712)
319 let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
320
321 let mut proto = commit_to_proto(&commit);
322 if let Some(sign) = sign_with {
323 let data = proto.encode_to_vec();
324 let sig = sign(&data).map_err(to_other_err)?;
325 proto.secure_sig = Some(sig.clone());
326 commit.secure_sig = Some(SecureSig { data, sig });
327 }
328
329 temp_file
330 .as_file()
331 .write_all(&proto.encode_to_vec())
332 .map_err(to_other_err)?;
333
334 let id = CommitId::new(blake2b_hash(&commit).to_vec());
335
336 persist_content_addressed_temp_file(temp_file, self.commit_path(&id))
337 .map_err(to_other_err)?;
338 Ok((id, commit))
339 }
340
341 fn get_copy_records(
342 &self,
343 _paths: Option<&[RepoPathBuf]>,
344 _root: &CommitId,
345 _head: &CommitId,
346 ) -> BackendResult<BoxStream<BackendResult<CopyRecord>>> {
347 Ok(Box::pin(stream::empty()))
348 }
349
350 fn gc(&self, _index: &dyn Index, _keep_newer: SystemTime) -> BackendResult<()> {
351 Ok(())
352 }
353}
354
355#[allow(clippy::assigning_clones)]
356pub fn commit_to_proto(commit: &Commit) -> crate::protos::simple_store::Commit {
357 let mut proto = crate::protos::simple_store::Commit::default();
358 for parent in &commit.parents {
359 proto.parents.push(parent.to_bytes());
360 }
361 for predecessor in &commit.predecessors {
362 proto.predecessors.push(predecessor.to_bytes());
363 }
364 match &commit.root_tree {
365 MergedTreeId::Legacy(tree_id) => {
366 proto.root_tree = vec![tree_id.to_bytes()];
367 }
368 MergedTreeId::Merge(tree_ids) => {
369 proto.uses_tree_conflict_format = true;
370 proto.root_tree = tree_ids.iter().map(|id| id.to_bytes()).collect();
371 }
372 }
373 proto.change_id = commit.change_id.to_bytes();
374 proto.description = commit.description.clone();
375 proto.author = Some(signature_to_proto(&commit.author));
376 proto.committer = Some(signature_to_proto(&commit.committer));
377 proto
378}
379
380fn commit_from_proto(mut proto: crate::protos::simple_store::Commit) -> Commit {
381 // Note how .take() sets the secure_sig field to None before we encode the data.
382 // Needs to be done first since proto is partially moved a bunch below
383 let secure_sig = proto.secure_sig.take().map(|sig| SecureSig {
384 data: proto.encode_to_vec(),
385 sig,
386 });
387
388 let parents = proto.parents.into_iter().map(CommitId::new).collect();
389 let predecessors = proto.predecessors.into_iter().map(CommitId::new).collect();
390 let root_tree = if proto.uses_tree_conflict_format {
391 let merge_builder: MergeBuilder<_> = proto.root_tree.into_iter().map(TreeId::new).collect();
392 MergedTreeId::Merge(merge_builder.build())
393 } else {
394 assert_eq!(proto.root_tree.len(), 1);
395 MergedTreeId::Legacy(TreeId::new(proto.root_tree[0].clone()))
396 };
397 let change_id = ChangeId::new(proto.change_id);
398 Commit {
399 parents,
400 predecessors,
401 root_tree,
402 change_id,
403 description: proto.description,
404 author: signature_from_proto(proto.author.unwrap_or_default()),
405 committer: signature_from_proto(proto.committer.unwrap_or_default()),
406 secure_sig,
407 }
408}
409
410fn tree_to_proto(tree: &Tree) -> crate::protos::simple_store::Tree {
411 let mut proto = crate::protos::simple_store::Tree::default();
412 for entry in tree.entries() {
413 proto
414 .entries
415 .push(crate::protos::simple_store::tree::Entry {
416 name: entry.name().as_internal_str().to_owned(),
417 value: Some(tree_value_to_proto(entry.value())),
418 });
419 }
420 proto
421}
422
423fn tree_from_proto(proto: crate::protos::simple_store::Tree) -> Tree {
424 let mut tree = Tree::default();
425 for proto_entry in proto.entries {
426 let value = tree_value_from_proto(proto_entry.value.unwrap());
427 tree.set(RepoPathComponentBuf::from(proto_entry.name), value);
428 }
429 tree
430}
431
432fn tree_value_to_proto(value: &TreeValue) -> crate::protos::simple_store::TreeValue {
433 let mut proto = crate::protos::simple_store::TreeValue::default();
434 match value {
435 TreeValue::File { id, executable } => {
436 proto.value = Some(crate::protos::simple_store::tree_value::Value::File(
437 crate::protos::simple_store::tree_value::File {
438 id: id.to_bytes(),
439 executable: *executable,
440 },
441 ));
442 }
443 TreeValue::Symlink(id) => {
444 proto.value = Some(crate::protos::simple_store::tree_value::Value::SymlinkId(
445 id.to_bytes(),
446 ));
447 }
448 TreeValue::GitSubmodule(_id) => {
449 panic!("cannot store git submodules");
450 }
451 TreeValue::Tree(id) => {
452 proto.value = Some(crate::protos::simple_store::tree_value::Value::TreeId(
453 id.to_bytes(),
454 ));
455 }
456 TreeValue::Conflict(id) => {
457 proto.value = Some(crate::protos::simple_store::tree_value::Value::ConflictId(
458 id.to_bytes(),
459 ));
460 }
461 }
462 proto
463}
464
465fn tree_value_from_proto(proto: crate::protos::simple_store::TreeValue) -> TreeValue {
466 match proto.value.unwrap() {
467 crate::protos::simple_store::tree_value::Value::TreeId(id) => {
468 TreeValue::Tree(TreeId::new(id))
469 }
470 crate::protos::simple_store::tree_value::Value::File(
471 crate::protos::simple_store::tree_value::File { id, executable, .. },
472 ) => TreeValue::File {
473 id: FileId::new(id),
474 executable,
475 },
476 crate::protos::simple_store::tree_value::Value::SymlinkId(id) => {
477 TreeValue::Symlink(SymlinkId::new(id))
478 }
479 crate::protos::simple_store::tree_value::Value::ConflictId(id) => {
480 TreeValue::Conflict(ConflictId::new(id))
481 }
482 }
483}
484
485fn signature_to_proto(signature: &Signature) -> crate::protos::simple_store::commit::Signature {
486 crate::protos::simple_store::commit::Signature {
487 name: signature.name.clone(),
488 email: signature.email.clone(),
489 timestamp: Some(crate::protos::simple_store::commit::Timestamp {
490 millis_since_epoch: signature.timestamp.timestamp.0,
491 tz_offset: signature.timestamp.tz_offset,
492 }),
493 }
494}
495
496fn signature_from_proto(proto: crate::protos::simple_store::commit::Signature) -> Signature {
497 let timestamp = proto.timestamp.unwrap_or_default();
498 Signature {
499 name: proto.name,
500 email: proto.email,
501 timestamp: Timestamp {
502 timestamp: MillisSinceEpoch(timestamp.millis_since_epoch),
503 tz_offset: timestamp.tz_offset,
504 },
505 }
506}
507
508fn conflict_to_proto(conflict: &Conflict) -> crate::protos::simple_store::Conflict {
509 let mut proto = crate::protos::simple_store::Conflict::default();
510 for term in &conflict.removes {
511 proto.removes.push(conflict_term_to_proto(term));
512 }
513 for term in &conflict.adds {
514 proto.adds.push(conflict_term_to_proto(term));
515 }
516 proto
517}
518
519fn conflict_from_proto(proto: crate::protos::simple_store::Conflict) -> Conflict {
520 let removes = proto
521 .removes
522 .into_iter()
523 .map(conflict_term_from_proto)
524 .collect();
525 let adds = proto
526 .adds
527 .into_iter()
528 .map(conflict_term_from_proto)
529 .collect();
530 Conflict { removes, adds }
531}
532
533fn conflict_term_from_proto(proto: crate::protos::simple_store::conflict::Term) -> ConflictTerm {
534 ConflictTerm {
535 value: tree_value_from_proto(proto.content.unwrap()),
536 }
537}
538
539fn conflict_term_to_proto(part: &ConflictTerm) -> crate::protos::simple_store::conflict::Term {
540 crate::protos::simple_store::conflict::Term {
541 content: Some(tree_value_to_proto(&part.value)),
542 }
543}
544
545#[cfg(test)]
546mod tests {
547 use assert_matches::assert_matches;
548 use pollster::FutureExt as _;
549
550 use super::*;
551 use crate::tests::new_temp_dir;
552
553 /// Test that parents get written correctly
554 #[test]
555 fn write_commit_parents() {
556 let temp_dir = new_temp_dir();
557 let store_path = temp_dir.path();
558
559 let backend = SimpleBackend::init(store_path);
560 let mut commit = Commit {
561 parents: vec![],
562 predecessors: vec![],
563 root_tree: MergedTreeId::resolved(backend.empty_tree_id().clone()),
564 change_id: ChangeId::from_hex("abc123"),
565 description: "".to_string(),
566 author: create_signature(),
567 committer: create_signature(),
568 secure_sig: None,
569 };
570
571 let write_commit = |commit: Commit| -> BackendResult<(CommitId, Commit)> {
572 backend.write_commit(commit, None).block_on()
573 };
574
575 // No parents
576 commit.parents = vec![];
577 assert_matches!(
578 write_commit(commit.clone()),
579 Err(BackendError::Other(err)) if err.to_string().contains("no parents")
580 );
581
582 // Only root commit as parent
583 commit.parents = vec![backend.root_commit_id().clone()];
584 let first_id = write_commit(commit.clone()).unwrap().0;
585 let first_commit = backend.read_commit(&first_id).block_on().unwrap();
586 assert_eq!(first_commit, commit);
587
588 // Only non-root commit as parent
589 commit.parents = vec![first_id.clone()];
590 let second_id = write_commit(commit.clone()).unwrap().0;
591 let second_commit = backend.read_commit(&second_id).block_on().unwrap();
592 assert_eq!(second_commit, commit);
593
594 // Merge commit
595 commit.parents = vec![first_id.clone(), second_id.clone()];
596 let merge_id = write_commit(commit.clone()).unwrap().0;
597 let merge_commit = backend.read_commit(&merge_id).block_on().unwrap();
598 assert_eq!(merge_commit, commit);
599
600 // Merge commit with root as one parent
601 commit.parents = vec![first_id, backend.root_commit_id().clone()];
602 let root_merge_id = write_commit(commit.clone()).unwrap().0;
603 let root_merge_commit = backend.read_commit(&root_merge_id).block_on().unwrap();
604 assert_eq!(root_merge_commit, commit);
605 }
606
607 fn create_signature() -> Signature {
608 Signature {
609 name: "Someone".to_string(),
610 email: "someone@example.com".to_string(),
611 timestamp: Timestamp {
612 timestamp: MillisSinceEpoch(0),
613 tz_offset: 0,
614 },
615 }
616 }
617}