f18afb7 node: Change storage layout — heartwood

Radicle Heartwood Protocol & Stack

node: Change storage layout

Alexis Sellier committed 3 years ago

commit f18afb793c52fb5501546fe33a8d335aad8ba7b5
parent 5c41af060f16d8c47bb4c26b881fec91c93fa268

8 files changed +158 -259

modified node/src/git.rs

@@ -1,85 +1,62 @@

 use crate::identity::ProjId;
 use crate::storage::{Error, WriteStorage};
 use std::str::FromStr;
 use crate::collections::HashMap;
 use crate::identity::UserId;
 use crate::storage::{Remote, Remotes, Unverified};
 use git_ref_format as format;
 /// Default port of the `git` transport protocol.
 pub const PROTOCOL_PORT: u16 = 9418;
 /// Fetch all remotes of a project from the given URL.
 pub fn fetch<S: WriteStorage>(proj: &ProjId, url: &str, mut storage: S) -> Result<(), Error> {
     // TODO: Use `Url` type?
     // TODO: Have function to fetch specific remotes.
     // TODO: Return meaningful info on success.
     //
     // Repository layout should look like this:
     //
     //      /refs/namespaces/<project>
     //              /refs/namespaces/<remote>
     //                    /heads
     //                      /master
     //                    /tags
     //                    ...
     //
     let repo = storage.repository();
     let refs: &[&str] = &[&format!(
         "refs/namespaces/{}/refs/*:refs/namespaces/{}/refs/*",
         proj, proj
     )];
     let mut remote = repo.remote_anonymous(url)?;
     let mut opts = git2::FetchOptions::default();
     remote.fetch(refs, Some(&mut opts), None)?;
     Ok(())
 #[derive(thiserror::Error, Debug)]
 pub enum RefError {
     #[error("invalid ref name '{0}'")]
     InvalidName(format::RefString),
     #[error("invalid ref format: {0}")]
     Format(#[from] format::Error),
 }
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::hash::Digest;
     use crate::identity::ProjId;
     use crate::storage::Storage;
 #[derive(thiserror::Error, Debug)]
 pub enum ListRefsError {
     #[error("git error: {0}")]
     Git(#[from] git2::Error),
     #[error("invalid ref: {0}")]
     InvalidRef(#[from] RefError),
 }
     /// Create an initial empty commit.
     fn initial_commit(repo: &git2::Repository) -> Result<git2::Oid, Error> {
         // First use the config to initialize a commit signature for the user.
         let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?;
         // Now let's create an empty tree for this commit.
         let tree_id = repo.index()?.write_tree()?;
         let tree = repo.find_tree(tree_id)?;
         let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?;
 /// List remote refs of a project, given the remote URL.
 pub fn list_remotes(url: &str) -> Result<Remotes<Unverified>, ListRefsError> {
     let mut remotes = HashMap::default();
     let mut remote = git2::Remote::create_detached(url)?;
         Ok(oid)
     }
     remote.connect(git2::Direction::Fetch)?;
     #[test]
     fn test_fetch() {
         let path = tempfile::tempdir().unwrap().into_path();
         let alice = git2::Repository::init_bare(path.join("alice")).unwrap();
         let bob = git2::Repository::init_bare(path.join("bob")).unwrap();
         let mut bob_storage = Storage::from(bob);
         let proj = ProjId::from(Digest::new(&[42]));
         let master = format!("refs/namespaces/{}/refs/heads/master", proj);
         let alice_oid = initial_commit(&alice).unwrap();
     let refs = remote.list()?;
     for r in refs {
         let (id, refname) = parse_ref::<UserId>(r.name())?;
         let entry = remotes
             .entry(id.clone())
             .or_insert_with(|| Remote::new(id, HashMap::default()));
         alice
             .reference(&master, alice_oid, false, "Create master branch")
             .unwrap();
         // Have Bob fetch Alice's refs.
         fetch(
             &proj,
             &format!("file://{}/alice", path.display()),
             &mut bob_storage,
         )
         .unwrap();
         entry.refs.insert(refname.to_string(), r.oid().into());
     }
         let bob_oid = bob_storage
             .repository()
             .find_reference(&master)
             .unwrap()
             .target()
             .unwrap();
     Ok(Remotes::new(remotes))
 }
         assert_eq!(alice_oid, bob_oid);
     }
 /// Parse a ref string.
 pub fn parse_ref<T: FromStr>(s: &str) -> Result<(T, format::RefString), RefError> {
     let input = format::RefStr::try_from_str(s)?;
     let suffix = input
         .strip_prefix(format::refname!("refs/namespaces"))
         .ok_or_else(|| RefError::InvalidName(input.to_owned()))?;
     let mut components = suffix.components();
     let id = components
         .next()
         .ok_or_else(|| RefError::InvalidName(input.to_owned()))?;
     let id = T::from_str(&id.to_string()).map_err(|_| RefError::InvalidName(input.to_owned()))?;
     let refstr = components.collect::<format::RefString>();
     Ok((id, refstr))
 }

modified node/src/identity.rs

@@ -1,4 +1,4 @@

 use std::{fmt, io, ops::Deref, str::FromStr};
 use std::{ffi::OsString, fmt, io, ops::Deref, str::FromStr};
 use ed25519_consensus::{VerificationKey, VerificationKeyBytes};
 use nonempty::NonEmpty;

@@ -10,8 +10,6 @@ use crate::hash;

 #[derive(Error, Debug)]
 pub enum ProjIdError {
     #[error("invalid ref '{0}'")]
     InvalidRef(String),
     #[error("invalid digest: {0}")]
     InvalidDigest(#[from] hash::DecodeError),
 }

@@ -35,24 +33,25 @@ impl ProjId {

     pub fn encode(&self) -> String {
         multibase::encode(multibase::Base::Base58Btc, &self.0.as_ref())
     }
     pub(crate) fn from_ref(s: &str) -> Result<ProjId, ProjIdError> {
         if let Some(s) = s.split('/').nth(2) {
             let id = Self::from_str(s)?;
             return Ok(id);
         }
         Err(ProjIdError::InvalidRef(s.to_owned()))
     }
 }
 impl FromStr for ProjId {
     type Err = hash::DecodeError;
     type Err = ProjIdError;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(Self(hash::Digest::from_str(s)?))
     }
 }
 impl TryFrom<OsString> for ProjId {
     type Error = ProjIdError;
     fn try_from(value: OsString) -> Result<Self, Self::Error> {
         let string = value.to_string_lossy();
         Self::from_str(&string)
     }
 }
 impl From<hash::Digest> for ProjId {
     fn from(digest: hash::Digest) -> Self {
         Self(digest)

@@ -134,8 +133,6 @@ pub enum UserIdError {

     InvalidKey(#[from] ed25519_consensus::Error),
 }
 impl UserId {}
 #[derive(Error, Debug)]
 pub enum DocError {
     #[error("toml: {0}")]

modified node/src/protocol.rs

@@ -16,9 +16,8 @@ use crate::address_manager::AddressManager;

 use crate::clock::RefClock;
 use crate::collections::{HashMap, HashSet};
 use crate::decoder::Decoder;
 use crate::git;
 use crate::identity::{ProjId, UserId};
 use crate::storage;
 use crate::storage::{self, WriteRepository};
 use crate::storage::{Inventory, ReadStorage, Remotes, Unverified, WriteStorage};
 /// Network peer identifier.

@@ -452,7 +451,11 @@ where

         match cmd {
             Command::Connect(addr) => self.context.connect(addr),
             Command::Fetch(proj, remote) => {
                 git::fetch(&proj, &format!("git://{}", remote), &mut self.storage).unwrap();
                 self.storage
                     .repository(&proj)
                     .unwrap()
                     .fetch(&format!("git://{}", remote))
                     .unwrap();
             }
         }
     }

modified node/src/storage.rs

@@ -1,22 +1,22 @@

 pub mod git;
 use std::collections::hash_map;
 use std::io;
 use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut};
 use std::path::Path;
 use std::{fmt, fs, io};
 use git_ref_format::refspec;
 use once_cell::sync::Lazy;
 use radicle_git_ext as git_ext;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 pub use radicle_git_ext::Oid;
 use crate::collections::HashMap;
 use crate::git::RefError;
 use crate::identity;
 use crate::identity::{ProjId, ProjIdError, UserId};
 pub static RAD_ID_GLOB: Lazy<refspec::PatternString> =
     Lazy::new(|| refspec::pattern!("refs/namespaces/*/refs/rad/id"));
 pub static IDENTITY_PATH: Lazy<&Path> = Lazy::new(|| Path::new(".rad/identity.toml"));
 pub type BranchName = String;

@@ -27,6 +27,8 @@ pub type Inventory = Vec<(ProjId, HashMap<String, Remote<Unverified>>)>;

 pub enum Error {
     #[error("invalid git reference")]
     InvalidRef,
     #[error("git reference error: {0}")]
     Ref(#[from] RefError),
     #[error("git: {0}")]
     Git(#[from] git2::Error),
     #[error("id: {0}")]

@@ -51,7 +53,7 @@ pub struct Verified;

 pub struct Unverified;
 /// Project remotes. Tracks the git state of a project.
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Remotes<V>(HashMap<RemoteId, Remote<V>>);
 impl Remotes<Unverified> {

@@ -60,6 +62,21 @@ impl Remotes<Unverified> {

     }
 }
 impl Default for Remotes<Unverified> {
     fn default() -> Self {
         Self(HashMap::default())
     }
 }
 impl<V> IntoIterator for Remotes<V> {
     type Item = (RemoteId, Remote<V>);
     type IntoIter = hash_map::IntoIter<RemoteId, Remote<V>>;
     fn into_iter(self) -> Self::IntoIter {
         self.0.into_iter()
     }
 }
 #[allow(clippy::from_over_into)]
 impl Into<HashMap<String, Remote<Unverified>>> for Remotes<Unverified> {
     fn into(self) -> HashMap<String, Remote<Unverified>> {

@@ -73,20 +90,23 @@ impl Into<HashMap<String, Remote<Unverified>>> for Remotes<Unverified> {

 }
 /// A project remote.
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)]
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
 pub struct Remote<V> {
     /// ID of remote.
     pub id: UserId,
     /// Git references published under this remote, and their hashes.
     refs: HashMap<RefName, Oid>,
     pub refs: HashMap<RefName, Oid>,
     /// Whether this remote is of a project delegate.
     delegate: bool,
     pub delegate: bool,
     /// Whether the remote is verified or not, ie. whether its signed refs were checked.
     #[serde(skip)]
     verified: PhantomData<V>,
 }
 impl Remote<Unverified> {
     pub fn new(refs: HashMap<RefName, Oid>) -> Self {
     pub fn new(id: UserId, refs: HashMap<RefName, Oid>) -> Self {
         Self {
             id,
             refs,
             delegate: false,
             verified: PhantomData,

@@ -100,12 +120,18 @@ pub trait ReadStorage {

 }
 pub trait WriteStorage {
     fn repository(&mut self) -> &mut git2::Repository;
     fn namespace(
         &mut self,
         proj: &ProjId,
         user: &UserId,
     ) -> Result<&mut git2::Repository, git2::Error>;
     type Repository: WriteRepository;
     fn repository(&self, proj: &ProjId) -> Result<Self::Repository, Error>;
 }
 pub trait ReadRepository {
     fn remotes(&self) -> Result<Remotes<Unverified>, Error>;
 }
 pub trait WriteRepository {
     fn fetch(&mut self, url: &str) -> Result<(), git2::Error>;
     fn namespace(&mut self, user: &UserId) -> Result<&mut git2::Repository, git2::Error>;
 }
 impl<T, S> ReadStorage for T

@@ -127,114 +153,10 @@ where

     T: DerefMut<Target = S>,
     S: WriteStorage + 'static,
 {
     fn repository(&mut self) -> &mut git2::Repository {
         self.deref_mut().repository()
     }
     fn namespace(
         &mut self,
         proj: &ProjId,
         user: &UserId,
     ) -> Result<&mut git2::Repository, git2::Error> {
         self.deref_mut().namespace(proj, user)
     }
 }
 pub struct Storage {
     backend: git2::Repository,
 }
 impl From<git2::Repository> for Storage {
     fn from(backend: git2::Repository) -> Self {
         Self { backend }
     }
 }
 impl fmt::Debug for Storage {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "Storage(..)")
     }
 }
 impl ReadStorage for Storage {
     fn get(&self, _id: &ProjId) -> Result<Option<Remotes<Unverified>>, Error> {
         todo!()
     }
     fn inventory(&self) -> Result<Inventory, Error> {
         let glob: String = RAD_ID_GLOB.clone().into();
         let refs = self.backend.references_glob(glob.as_str())?;
         for r in refs {
             let r = r?;
             let name = r.name().ok_or(Error::InvalidRef)?;
             let _id = ProjId::from_ref(name)?;
             todo!();
         }
         Ok(vec![])
     }
 }
 impl WriteStorage for Storage {
     fn repository(&mut self) -> &mut git2::Repository {
         &mut self.backend
     }
     fn namespace(
         &mut self,
         proj: &ProjId,
         user: &UserId,
     ) -> Result<&mut git2::Repository, git2::Error> {
         let path = self.backend.path();
         self.backend = git2::Repository::open_bare(path)?;
         self.backend.set_namespace(&format!("{}/{}", proj, user))?;
         Ok(&mut self.backend)
     }
 }
 impl Storage {
     pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, git2::Error> {
         let path = path.as_ref();
         let backend = match git2::Repository::open_bare(path) {
             Err(e) if git_ext::is_not_found_err(&e) => {
                 let backend = git2::Repository::init_opts(
                     path,
                     git2::RepositoryInitOptions::new()
                         .bare(true)
                         .no_reinit(true)
                         .external_template(false),
                 )?;
                 Ok(backend)
             }
             Ok(repo) => Ok(repo),
             Err(e) => Err(e),
         }?;
         Ok(Self { backend })
     }
     type Repository = S::Repository;
     pub fn create(
         &self,
         repo: &git2::Repository,
         identity: impl Into<identity::Doc>,
     ) -> Result<(ProjId, git2::Reference), Error> {
         let doc = identity.into();
         let file = fs::OpenOptions::new()
             .create_new(true)
             .write(true)
             .open(*IDENTITY_PATH)?;
         let id = doc.write(file)?;
         let ref_name = RAD_ID_GLOB.replace('*', &id.encode());
         let oid = repo.head()?.target().ok_or(Error::InvalidHead)?;
         let reference = self.backend.reference(&ref_name, oid, false, "")?;
         // TODO: Push project to monorepo.
         Ok((id, reference))
     fn repository(&self, proj: &ProjId) -> Result<Self::Repository, Error> {
         self.deref().repository(proj)
     }
 }

modified node/src/storage/git.rs

@@ -1,29 +1,26 @@

 use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::{fmt, fs, io};
 use std::{fmt, fs};
 use git_ref_format::refspec;
 use git_url::Url;
 use once_cell::sync::Lazy;
 use radicle_git_ext as git_ext;
 use serde::{Deserialize, Serialize};
 pub use radicle_git_ext::Oid;
 use crate::collections::HashMap;
 use crate::git;
 use crate::identity;
 use crate::identity::{ProjId, ProjIdError, UserId};
 use crate::identity::{ProjId, UserId};
 use super::{
     Error, Inventory, ReadRepository, ReadStorage, Remote, Remotes, Unverified, Verified,
     WriteRepository, WriteStorage,
     Error, Inventory, ReadRepository, ReadStorage, Remote, Remotes, Unverified, WriteRepository,
     WriteStorage,
 };
 pub static RAD_ROOT_GLOB: Lazy<refspec::PatternString> =
     Lazy::new(|| refspec::pattern!("refs/namespaces/*/refs/rad/root"));
 pub static NAMESPACES_GLOB: Lazy<refspec::PatternString> =
     Lazy::new(|| refspec::pattern!("refs/namespaces/*"));
 pub static IDENTITY_PATH: Lazy<&Path> = Lazy::new(|| Path::new(".rad/identity.toml"));
 pub struct Storage {

@@ -48,7 +45,6 @@ impl ReadStorage for Storage {

     }
     fn inventory(&self) -> Result<Inventory, Error> {
         let glob: String = RAD_ROOT_GLOB.clone().into();
         let projs = self.projects()?;
         let mut inv = Vec::new();

@@ -57,7 +53,7 @@ impl ReadStorage for Storage {

             let remotes = repo
                 .remotes()?
                 .into_iter()
                 .map(|r| (r.id.to_string(), r))
                 .map(|(id, r)| (id.to_string(), r))
                 .collect();
             inv.push((proj, remotes));

@@ -157,8 +153,8 @@ impl Repository {

 }
 impl ReadRepository for Repository {
     fn remotes(&self) -> Result<Vec<Remote<Unverified>>, Error> {
         let refs = self.backend.references_glob(RAD_ROOT_GLOB.as_str())?;
     fn remotes(&self) -> Result<Remotes<Unverified>, Error> {
         let refs = self.backend.references_glob(NAMESPACES_GLOB.as_str())?;
         let mut remotes = HashMap::default();
         for r in refs {

@@ -172,7 +168,7 @@ impl ReadRepository for Repository {

             entry.refs.insert(refname.to_string(), oid.into());
         }
         Ok(remotes.into_values().collect())
         Ok(Remotes::new(remotes))
     }
 }

@@ -192,7 +188,7 @@ impl WriteRepository for Repository {

         //                    /tags
         //                    ...
         //
         let refs: &[&str] = &[&format!("refs/namespaces/*:refs/namespaces/*")];
         let refs: &[&str] = &["refs/namespaces/*:refs/namespaces/*"];
         let mut remote = self.backend.remote_anonymous(url)?;
         let mut opts = git2::FetchOptions::default();

@@ -221,38 +217,22 @@ impl From<git2::Repository> for Repository {

 mod tests {
     use super::*;
     use crate::git;
     use crate::hash::Digest;
     use crate::identity::ProjId;
     use crate::storage::{ReadStorage, WriteRepository};
     use crate::test::fixtures;
     /// Create an initial empty commit.
     fn initial_commit(repo: &git2::Repository) -> Result<git2::Oid, Error> {
         // First use the config to initialize a commit signature for the user.
         let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?;
         // Now let's create an empty tree for this commit.
         let tree_id = repo.index()?.write_tree()?;
         let tree = repo.find_tree(tree_id)?;
         let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?;
         Ok(oid)
     }
     #[test]
     fn test_ls_remote() {
         crate::test::logger::init(log::Level::Debug);
     fn test_list_remotes() {
         let dir = tempfile::tempdir().unwrap();
         let storage = fixtures::storage(dir.path());
         let inv = storage.inventory().unwrap();
         let (proj, _) = inv.first().unwrap();
         let refs = git::list_refs(&format!(
         let refs = git::list_remotes(&format!(
             "file://{}",
             dir.path().join(&proj.to_string()).display(),
         ))
         .unwrap();
         let remotes = storage.repository(&proj).unwrap().remotes().unwrap();
         let remotes = storage.repository(proj).unwrap().remotes().unwrap();
         assert_eq!(refs, remotes);
     }

@@ -267,7 +247,7 @@ mod tests {

         let refname = "refs/heads/master";
         // Have Bob fetch Alice's refs.
         bob.repository(&proj)
         bob.repository(proj)
             .unwrap()
             .fetch(&format!(
                 "file://{}",

@@ -275,14 +255,14 @@ mod tests {

             ))
             .unwrap();
         for (_, remote) in remotes {
         for remote in remotes.values() {
             let alice_oid = alice
                 .repository(&proj)
                 .repository(proj)
                 .unwrap()
                 .find_reference(&remote.id, refname)
                 .unwrap();
             let bob_oid = bob
                 .repository(&proj)
                 .repository(proj)
                 .unwrap()
                 .find_reference(&remote.id, refname)
                 .unwrap();

modified node/src/test/arbitrary.rs

@@ -26,6 +26,7 @@ impl quickcheck::Arbitrary for storage::Remote<storage::Unverified> {

         let mut refs: HashMap<storage::BranchName, storage::Oid> = HashMap::with_hasher(rng.into());
         let mut bytes: [u8; 20] = [0; 20];
         let names = &["master", "dev", "feature/1", "feature/2", "feature/3"];
         let id = UserId::arbitrary(g);
         for _ in 0..g.size().min(2) {
             if let Some(name) = g.choose(names) {

@@ -36,7 +37,7 @@ impl quickcheck::Arbitrary for storage::Remote<storage::Unverified> {

                 refs.insert(name.to_string(), oid);
             }
         }
         storage::Remote::new(refs)
         storage::Remote::new(id, refs)
     }
 }

modified node/src/test/fixtures.rs

@@ -4,7 +4,8 @@ use std::str::FromStr;

 use once_cell::sync::Lazy;
 use crate::identity::{ProjId, UserId};
 use crate::storage::{Storage, WriteStorage};
 use crate::storage::git::Storage;
 use crate::storage::{WriteRepository, WriteStorage};
 pub static USER_IDS: Lazy<[UserId; 16]> = Lazy::new(|| {
     [

@@ -48,16 +49,23 @@ pub static PROJ_IDS: Lazy<[ProjId; 16]> = Lazy::new(|| {

     ]
 });
 pub fn storage(path: &Path) -> Storage {
     let mut storage = Storage::open(path).unwrap();
 pub fn storage<P: AsRef<Path>>(path: P) -> Storage {
     let path = path.as_ref();
     let storage = Storage::new(path);
     for proj in PROJ_IDS.iter().take(3) {
         log::debug!("creating {}...", proj);
         let mut repo = storage.repository(proj).unwrap();
         for user in USER_IDS.iter().take(3) {
             let repo = storage.namespace(proj, user).unwrap();
             let repo = repo.namespace(user).unwrap();
             let head_oid = initial_commit(repo).unwrap();
             let head = repo.find_commit(head_oid).unwrap();
             log::debug!("creating {}...", repo.namespace().unwrap());
             log::debug!("{}: creating {}...", proj, repo.namespace().unwrap());
             repo.reference("refs/rad/root", head_oid, false, "test")
                 .unwrap();
             // TODO: Different commits.
             repo.branch("master", &head, false).unwrap();

modified node/src/test/storage.rs

@@ -1,5 +1,7 @@

 use crate::identity::ProjId;
 use crate::storage::{Error, Inventory, ReadStorage, Remotes, Unverified, WriteStorage};
 use crate::storage::{
     Error, Inventory, ReadStorage, Remotes, Unverified, WriteRepository, WriteStorage,
 };
 #[derive(Clone, Debug)]
 pub struct MockStorage {

@@ -38,13 +40,22 @@ impl ReadStorage for MockStorage {

 }
 impl WriteStorage for MockStorage {
     fn repository(&mut self) -> &mut git2::Repository {
     type Repository = MockRepository;
     fn repository(&self, _proj: &ProjId) -> Result<Self::Repository, Error> {
         todo!()
     }
 }
 pub struct MockRepository {}
 impl WriteRepository for MockRepository {
     fn fetch(&mut self, _url: &str) -> Result<(), git2::Error> {
         todo!()
     }
     fn namespace(
         &mut self,
         _proj: &ProjId,
         _user: &crate::identity::UserId,
     ) -> Result<&mut git2::Repository, git2::Error> {
         todo!()