Radish alpha
h
Radicle Heartwood Protocol & Stack
Radicle
Git (anonymous pull)
Log in to clone via SSH
node: Get `git::fetch` working
Alexis Sellier committed 3 years ago
commit aa28e40f672fb60a16d8047ef16f573982c474e0
parent 9873a364638732d72e7e24df0c0a63ecd9c6adee
5 files changed +464 -13
modified node/src/git.rs
@@ -1,24 +1,85 @@
-
use std::net;
-

use crate::identity::ProjId;
use crate::storage::{Error, WriteStorage};

/// Default port of the `git` transport protocol.
pub const PROTOCOL_PORT: u16 = 9418;

-
pub fn fetch<S: WriteStorage>(
-
    _proj: &ProjId,
-
    remote: &net::SocketAddr,
-
    mut storage: S,
-
) -> Result<(), Error> {
-
    let _repo = storage.repository();
-

-
    let url = format!("git://{}", remote);
-
    let refs: &[&str] = &[];
-
    let mut remote = git2::Remote::create_detached(&url)?;
+
/// Fetch all remotes of a project from the given URL.
+
pub fn fetch<S: WriteStorage>(proj: &ProjId, url: &str, mut storage: S) -> Result<(), Error> {
+
    // TODO: Use `Url` type?
+
    // TODO: Have function to fetch specific remotes.
+
    // TODO: Return meaningful info on success.
+
    //
+
    // Repository layout should look like this:
+
    //
+
    //      /refs/namespaces/<project>
+
    //              /refs/namespaces/<remote>
+
    //                    /heads
+
    //                      /master
+
    //                    /tags
+
    //                    ...
+
    //
+
    let repo = storage.repository();
+
    let refs: &[&str] = &[&format!(
+
        "refs/namespaces/{}/refs/*:refs/namespaces/{}/refs/*",
+
        proj, proj
+
    )];
+
    let mut remote = repo.remote_anonymous(url)?;
    let mut opts = git2::FetchOptions::default();

    remote.fetch(refs, Some(&mut opts), None)?;

    Ok(())
}
+

+
#[cfg(test)]
+
mod tests {
+
    use super::*;
+
    use crate::hash::Digest;
+
    use crate::identity::ProjId;
+
    use crate::storage::Storage;
+

+
    /// Create an initial empty commit.
+
    fn initial_commit(repo: &git2::Repository) -> Result<git2::Oid, Error> {
+
        // First use the config to initialize a commit signature for the user.
+
        let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?;
+
        // Now let's create an empty tree for this commit.
+
        let tree_id = repo.index()?.write_tree()?;
+
        let tree = repo.find_tree(tree_id)?;
+
        let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?;
+

+
        Ok(oid)
+
    }
+

+
    #[test]
+
    fn test_fetch() {
+
        let path = tempfile::tempdir().unwrap().into_path();
+
        let alice = git2::Repository::init_bare(path.join("alice")).unwrap();
+
        let bob = git2::Repository::init_bare(path.join("bob")).unwrap();
+
        let mut bob_storage = Storage::from(bob);
+
        let proj = ProjId::from(Digest::new(&[42]));
+
        let master = format!("refs/namespaces/{}/refs/heads/master", proj);
+
        let alice_oid = initial_commit(&alice).unwrap();
+

+
        alice
+
            .reference(&master, alice_oid, false, "Create master branch")
+
            .unwrap();
+

+
        // Have Bob fetch Alice's refs.
+
        fetch(
+
            &proj,
+
            &format!("file://{}/alice", path.display()),
+
            &mut bob_storage,
+
        )
+
        .unwrap();
+

+
        let bob_oid = bob_storage
+
            .repository()
+
            .find_reference(&master)
+
            .unwrap()
+
            .target()
+
            .unwrap();
+

+
        assert_eq!(alice_oid, bob_oid);
+
    }
+
}
modified node/src/protocol.rs
@@ -441,7 +441,7 @@ where
        match cmd {
            Command::Connect(addr) => self.context.connect(addr),
            Command::Fetch(proj, remote) => {
-
                git::fetch(&proj, &remote, &mut self.storage).unwrap();
+
                git::fetch(&proj, &format!("git://{}", remote), &mut self.storage).unwrap();
            }
        }
    }
modified node/src/storage.rs
@@ -131,6 +131,12 @@ pub struct Storage {
    backend: git2::Repository,
}

+
impl From<git2::Repository> for Storage {
+
    fn from(backend: git2::Repository) -> Self {
+
        Self { backend }
+
    }
+
}
+

impl fmt::Debug for Storage {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "Storage(..)")
added node/src/storage/git.rs
@@ -0,0 +1,293 @@
+
use std::marker::PhantomData;
+
use std::ops::{Deref, DerefMut};
+
use std::path::{Path, PathBuf};
+
use std::str::FromStr;
+
use std::{fmt, fs, io};
+

+
use git_ref_format::refspec;
+
use git_url::Url;
+
use once_cell::sync::Lazy;
+
use radicle_git_ext as git_ext;
+
use serde::{Deserialize, Serialize};
+

+
pub use radicle_git_ext::Oid;
+

+
use crate::collections::HashMap;
+
use crate::git;
+
use crate::identity;
+
use crate::identity::{ProjId, ProjIdError, UserId};
+

+
use super::{
+
    Error, Inventory, ReadRepository, ReadStorage, Remote, Remotes, Unverified, Verified,
+
    WriteRepository, WriteStorage,
+
};
+

+
pub static RAD_ROOT_GLOB: Lazy<refspec::PatternString> =
+
    Lazy::new(|| refspec::pattern!("refs/namespaces/*/refs/rad/root"));
+
pub static IDENTITY_PATH: Lazy<&Path> = Lazy::new(|| Path::new(".rad/identity.toml"));
+

+
pub struct Storage {
+
    path: PathBuf,
+
}
+

+
impl From<PathBuf> for Storage {
+
    fn from(path: PathBuf) -> Self {
+
        Self { path }
+
    }
+
}
+

+
impl fmt::Debug for Storage {
+
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+
        write!(f, "Storage(..)")
+
    }
+
}
+

+
impl ReadStorage for Storage {
+
    fn get(&self, _id: &ProjId) -> Result<Option<Remotes<Unverified>>, Error> {
+
        todo!()
+
    }
+

+
    fn inventory(&self) -> Result<Inventory, Error> {
+
        let glob: String = RAD_ROOT_GLOB.clone().into();
+
        let projs = self.projects()?;
+
        let mut inv = Vec::new();
+

+
        for proj in projs {
+
            let repo = self.repository(&proj)?;
+
            let remotes = repo
+
                .remotes()?
+
                .into_iter()
+
                .map(|r| (r.id.to_string(), r))
+
                .collect();
+

+
            inv.push((proj, remotes));
+
        }
+
        Ok(inv)
+
    }
+
}
+

+
impl WriteStorage for Storage {
+
    type Repository = Repository;
+

+
    fn repository(&self, proj: &ProjId) -> Result<Self::Repository, Error> {
+
        Repository::open(self.path.join(proj.to_string()))
+
    }
+
}
+

+
impl Storage {
+
    pub fn new<P: AsRef<Path>>(path: P) -> Self {
+
        let path = path.as_ref().to_path_buf();
+

+
        Self { path }
+
    }
+

+
    pub fn path(&self) -> &Path {
+
        self.path.as_path()
+
    }
+

+
    pub fn projects(&self) -> Result<Vec<ProjId>, Error> {
+
        let mut projects = Vec::new();
+

+
        for result in fs::read_dir(&self.path)? {
+
            let path = result?;
+
            let id = ProjId::try_from(path.file_name())?;
+

+
            projects.push(id);
+
        }
+
        Ok(projects)
+
    }
+

+
    pub fn create(
+
        &self,
+
        repo: &git2::Repository,
+
        identity: impl Into<identity::Doc>,
+
    ) -> Result<(ProjId, Oid), Error> {
+
        let doc = identity.into();
+
        let file = fs::OpenOptions::new()
+
            .create_new(true)
+
            .write(true)
+
            .open(*IDENTITY_PATH)?;
+
        let id = doc.write(file)?;
+
        let ref_name = RAD_ROOT_GLOB.replace('*', &id.encode());
+
        let oid = repo.head()?.target().ok_or(Error::InvalidHead)?;
+
        let repository = self.repository(&id)?;
+
        let _reference = repository.backend.reference(&ref_name, oid, false, "")?;
+

+
        // TODO: Push project to monorepo.
+

+
        Ok((id, oid.into()))
+
    }
+
}
+

+
pub struct Repository {
+
    backend: git2::Repository,
+
}
+

+
impl Repository {
+
    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
+
        let backend = match git2::Repository::open_bare(path.as_ref()) {
+
            Err(e) if git_ext::is_not_found_err(&e) => {
+
                let backend = git2::Repository::init_opts(
+
                    path,
+
                    git2::RepositoryInitOptions::new()
+
                        .bare(true)
+
                        .no_reinit(true)
+
                        .external_template(false),
+
                )?;
+

+
                Ok(backend)
+
            }
+
            Ok(repo) => Ok(repo),
+
            Err(e) => Err(e),
+
        }?;
+

+
        Ok(Self { backend })
+
    }
+

+
    pub fn find_reference(&self, remote: &UserId, name: &str) -> Result<Oid, Error> {
+
        let name = format!("refs/namespaces/{}/{}", remote, name);
+
        let target = self
+
            .backend
+
            .find_reference(&name)?
+
            .target()
+
            .ok_or(Error::InvalidRef)?;
+

+
        Ok(target.into())
+
    }
+
}
+

+
impl ReadRepository for Repository {
+
    fn remotes(&self) -> Result<Vec<Remote<Unverified>>, Error> {
+
        let refs = self.backend.references_glob(RAD_ROOT_GLOB.as_str())?;
+
        let mut remotes = HashMap::default();
+

+
        for r in refs {
+
            let r = r?;
+
            let name = r.name().ok_or(Error::InvalidRef)?;
+
            let (id, refname) = git::parse_ref::<UserId>(name)?;
+
            let entry = remotes
+
                .entry(id.clone())
+
                .or_insert_with(|| Remote::new(id, HashMap::default()));
+
            let oid = r.target().ok_or(Error::InvalidRef)?;
+

+
            entry.refs.insert(refname.to_string(), oid.into());
+
        }
+
        Ok(remotes.into_values().collect())
+
    }
+
}
+

+
impl WriteRepository for Repository {
+
    /// Fetch all remotes of a project from the given URL.
+
    fn fetch(&mut self, url: &str) -> Result<(), git2::Error> {
+
        // TODO: Use `Url` type?
+
        // TODO: Have function to fetch specific remotes.
+
        // TODO: Return meaningful info on success.
+
        //
+
        // Repository layout should look like this:
+
        //
+
        //      /refs/namespaces/<project>
+
        //              /refs/namespaces/<remote>
+
        //                    /heads
+
        //                      /master
+
        //                    /tags
+
        //                    ...
+
        //
+
        let refs: &[&str] = &[&format!("refs/namespaces/*:refs/namespaces/*")];
+
        let mut remote = self.backend.remote_anonymous(url)?;
+
        let mut opts = git2::FetchOptions::default();
+

+
        remote.fetch(refs, Some(&mut opts), None)?;
+

+
        Ok(())
+
    }
+

+
    fn namespace(&mut self, user: &UserId) -> Result<&mut git2::Repository, git2::Error> {
+
        let path = self.backend.path();
+

+
        self.backend = git2::Repository::open_bare(path)?;
+
        self.backend.set_namespace(&user.to_string())?;
+

+
        Ok(&mut self.backend)
+
    }
+
}
+

+
impl From<git2::Repository> for Repository {
+
    fn from(backend: git2::Repository) -> Self {
+
        Self { backend }
+
    }
+
}
+

+
#[cfg(test)]
+
mod tests {
+
    use super::*;
+
    use crate::git;
+
    use crate::hash::Digest;
+
    use crate::identity::ProjId;
+
    use crate::storage::{ReadStorage, WriteRepository};
+
    use crate::test::fixtures;
+

+
    /// Create an initial empty commit.
+
    fn initial_commit(repo: &git2::Repository) -> Result<git2::Oid, Error> {
+
        // First use the config to initialize a commit signature for the user.
+
        let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?;
+
        // Now let's create an empty tree for this commit.
+
        let tree_id = repo.index()?.write_tree()?;
+
        let tree = repo.find_tree(tree_id)?;
+
        let oid = repo.commit(Some("HEAD"), &sig, &sig, "Initial commit", &tree, &[])?;
+

+
        Ok(oid)
+
    }
+

+
    #[test]
+
    fn test_ls_remote() {
+
        crate::test::logger::init(log::Level::Debug);
+

+
        let dir = tempfile::tempdir().unwrap();
+
        let storage = fixtures::storage(dir.path());
+
        let inv = storage.inventory().unwrap();
+
        let (proj, _) = inv.first().unwrap();
+
        let refs = git::list_refs(&format!(
+
            "file://{}",
+
            dir.path().join(&proj.to_string()).display(),
+
        ))
+
        .unwrap();
+

+
        let remotes = storage.repository(&proj).unwrap().remotes().unwrap();
+

+
        assert_eq!(refs, remotes);
+
    }
+

+
    #[test]
+
    fn test_fetch() {
+
        let path = tempfile::tempdir().unwrap().into_path();
+
        let alice = fixtures::storage(path.join("alice"));
+
        let bob = Storage::new(path.join("bob"));
+
        let inventory = alice.inventory().unwrap();
+
        let (proj, remotes) = inventory.first().unwrap();
+
        let refname = "refs/heads/master";
+

+
        // Have Bob fetch Alice's refs.
+
        bob.repository(&proj)
+
            .unwrap()
+
            .fetch(&format!(
+
                "file://{}",
+
                alice.path().join(&proj.to_string()).display()
+
            ))
+
            .unwrap();
+

+
        for (_, remote) in remotes {
+
            let alice_oid = alice
+
                .repository(&proj)
+
                .unwrap()
+
                .find_reference(&remote.id, refname)
+
                .unwrap();
+
            let bob_oid = bob
+
                .repository(&proj)
+
                .unwrap()
+
                .find_reference(&remote.id, refname)
+
                .unwrap();
+

+
            assert_eq!(alice_oid, bob_oid);
+
        }
+
    }
+
}
added node/src/test/fixtures.rs
@@ -0,0 +1,91 @@
+
use std::path::Path;
+
use std::str::FromStr;
+

+
use once_cell::sync::Lazy;
+

+
use crate::identity::{ProjId, UserId};
+
use crate::storage::{Storage, WriteStorage};
+

+
pub static USER_IDS: Lazy<[UserId; 16]> = Lazy::new(|| {
+
    [
+
        UserId::from_str("zCBH2FXDERonR1rDopkZTKAZAFKpXkiFc46XHYz1Qcyb4").unwrap(),
+
        UserId::from_str("z68bx7oq3JX3d5RVQSZzh7S9qP6a7AFNQnRas8EiELeGm").unwrap(),
+
        UserId::from_str("z8QVc8haUs3rc23ZfQYcFdntDCE5TGVWGhB1Piit2FyLW").unwrap(),
+
        UserId::from_str("zAmTnRTXhk49nShSWDFvi93SMFgTRGKAAJyHpXJ4rAFb8").unwrap(),
+
        UserId::from_str("z4RqaY63zcPZY2UudyUxfUrWu2FpTLMrCCfskah6YWxww").unwrap(),
+
        UserId::from_str("zHvS7fNSn3b8kE9Vy83wuitRTqkyqvRC7G7q8B1kbxPfG").unwrap(),
+
        UserId::from_str("z9Q8YnVrkpTCEx4ffFgXhEMNSbwX7unMJYLLTNF78Vjd9").unwrap(),
+
        UserId::from_str("zBTfXzjQhNgob6f5D3rhpJjTpFzjbLzU85QoKdS3CgT6v").unwrap(),
+
        UserId::from_str("z8qQNwfQZqYPQp9xyDssZyh7QctGWAjvq7A2T8vE4oWLr").unwrap(),
+
        UserId::from_str("z5YzpLeMn6ozf95bELziodGNpyTs5jQ7ssfofdv4rhB92").unwrap(),
+
        UserId::from_str("z8YUtfXfp5bthrongT11C9fYcSsT6QKY1SgfENxB5KXvj").unwrap(),
+
        UserId::from_str("zBSUVcBSUWPtYWoPPxZg9QPDDTFVQe2dZaXLaCvFvd9Di").unwrap(),
+
        UserId::from_str("z6ba5eWTvR22JL4ej4qErnGcxJTZF3YCHEH9FMriQbsmj").unwrap(),
+
        UserId::from_str("z6MJB1N1WfwWzt69k39eRmHGZ7CCefo4h68zX1gBEWKyh").unwrap(),
+
        UserId::from_str("zD2UwYEK4FGcrX7HqAPnT5i42uYG6ZgeeSo3C52a21ktJ").unwrap(),
+
        UserId::from_str("z5NVinWZWpNz7EbU26mdZ2nQV3inK3ZHw3YuW8Dd6puyw").unwrap(),
+
    ]
+
});
+

+
pub static PROJ_IDS: Lazy<[ProjId; 16]> = Lazy::new(|| {
+
    [
+
        ProjId::from_str("z3VDhnNMUwoaQHxNm5iNMEYuwY6RRi1e6WZ74oJKAWJzS").unwrap(),
+
        ProjId::from_str("zZXpj5rW3GsGnBXhszTYN5hru45AoKdqc7rbD7KK23y2").unwrap(),
+
        ProjId::from_str("z35ytZY1YSnk2M7Riz9KbEVfdrWAbrLdLKP5bpgPZR1uM").unwrap(),
+
        ProjId::from_str("zJDZF6mV6g1owvYnqbc8rGzYxbigsP7SAMkFZZXnwwxyc").unwrap(),
+
        ProjId::from_str("z7vwbcQRR8nu3GaHSyxQd2AvQuVsiKEFtV4EoBZGGSECn").unwrap(),
+
        ProjId::from_str("zDNvLCdwYAsRzH2UQzb1D5CjV5xf2rDsARsFXUymqswJF").unwrap(),
+
        ProjId::from_str("zAo9SpyTcwYxSe3ReaZj42T2zAFik3gY2A2eJhchLrArA").unwrap(),
+
        ProjId::from_str("zFXbHCdxjJpYJ1rYW8T7qUoZnkMcPwxR3xARZZpSqdGgg").unwrap(),
+
        ProjId::from_str("zDT5dWPNUudw9T8gD2vBZ6RZ6tXjzgnvoj1UdMnCGHeFr").unwrap(),
+
        ProjId::from_str("z7aD9ReLj8RbchMJuVgy928oT764iWq7p4wKCwWqroH7V").unwrap(),
+
        ProjId::from_str("z3PRwULg4pDXpv5GJe2z563hM7YuJFQdWPbouFkpEfUoS").unwrap(),
+
        ProjId::from_str("zBTQxg8xG8gGqUFtJwWotft3QuFuPPhA1aEBFLhqHuX4c").unwrap(),
+
        ProjId::from_str("z8exxN2CzTJDcFC5n8CPzskrpUYdu7rzupwCzjTUgmtb4").unwrap(),
+
        ProjId::from_str("z5Fa2PbkXKMLnvf4ZEbdHDiTmKVcqKRgxZn8xwVBZHkn1").unwrap(),
+
        ProjId::from_str("zEG2mTq7ExW3wBEgjwUDHqDQXZgamyv8cDmZud3b27SaJ").unwrap(),
+
        ProjId::from_str("z3dCpDtvBFj55HLvEW3grgqiYeanNHxW1hYS1TSK8epS4").unwrap(),
+
    ]
+
});
+

+
pub fn storage(path: &Path) -> Storage {
+
    let mut storage = Storage::open(path).unwrap();
+

+
    for proj in PROJ_IDS.iter().take(3) {
+
        for user in USER_IDS.iter().take(3) {
+
            let repo = storage.namespace(proj, user).unwrap();
+
            let head_oid = initial_commit(repo).unwrap();
+
            let head = repo.find_commit(head_oid).unwrap();
+

+
            log::debug!("creating {}...", repo.namespace().unwrap());
+

+
            // TODO: Different commits.
+
            repo.branch("master", &head, false).unwrap();
+
            repo.branch("patch/3", &head, false).unwrap();
+
        }
+
    }
+
    storage
+
}
+

+
/// Create an initial empty commit.
+
fn initial_commit(repo: &git2::Repository) -> Result<git2::Oid, git2::Error> {
+
    let sig = git2::Signature::now("cloudhead", "cloudhead@radicle.xyz")?;
+
    // Now let's create an empty tree for this commit.
+
    let tree_id = repo.index()?.write_tree()?;
+
    let tree = repo.find_tree(tree_id)?;
+
    let oid = repo.commit(None, &sig, &sig, "Initial commit", &tree, &[])?;
+

+
    Ok(oid)
+
}
+

+
#[cfg(test)]
+
mod tests {
+
    use super::*;
+

+
    #[test]
+
    fn smoke() {
+
        let path = tempfile::tempdir().unwrap().into_path();
+

+
        storage(&path);
+
    }
+
}