6649342 Add 'local' and 'remote' URL formats — heartwood

Radicle Heartwood Protocol & Stack

Add 'local' and 'remote' URL formats

Alexis Sellier committed 3 years ago

commit 66493422c2d1f99d5ff9210f3ee8e72eb725a815
parent 7c65f08cd1935bc811912913ee87359a7fd44d9f

8 files changed +371 -227

modified radicle-remote-helper/src/lib.rs

@@ -7,7 +7,7 @@ use thiserror::Error;

 use radicle::crypto::{PublicKey, Signer};
 use radicle::node::Handle;
 use radicle::ssh;
 use radicle::storage::git::transport::{Url, UrlError};
 use radicle::storage::git::transport::local::{Url, UrlError};
 use radicle::storage::{ReadRepository, WriteRepository, WriteStorage};
 /// The service invoked by git on the remote repository, during a push.

@@ -52,11 +52,11 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +

         }
     }?;
     // Default to profile key.
     let public_key = url
         .public_key
     let namespace = url
         .namespace
         .unwrap_or_else(|| *profile.signer.public_key());
     let proj = profile.storage.repository(url.id)?;
     let proj = profile.storage.repository(url.repo)?;
     if proj.is_empty()? {
         return Err(Error::RepositoryNotFound(proj.path().to_path_buf()).into());
     }

@@ -84,14 +84,14 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +

                 // 2. Our key is not the one loaded in the profile, which means that the signed refs
                 //    won't match the remote we're pushing to.
                 if *service == GIT_RECEIVE_PACK {
                     if profile.signer.public_key() != &public_key {
                         return Err(Error::KeyMismatch(public_key).into());
                     if profile.signer.public_key() != &namespace {
                         return Err(Error::KeyMismatch(namespace).into());
                     }
                     if !ssh::agent::connect()?
                         .request_identities::<PublicKey>()?
                         .contains(&public_key)
                         .contains(&namespace)
                     {
                         return Err(Error::KeyNotRegistered(public_key).into());
                         return Err(Error::KeyNotRegistered(namespace).into());
                     }
                 }
                 println!(); // Empty line signifies connection is established.

@@ -99,7 +99,7 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +

                 let mut child = process::Command::new(service)
                     .arg(proj.path())
                     .env("GIT_DIR", proj.path())
                     .env("GIT_NAMESPACE", public_key.to_string())
                     .env("GIT_NAMESPACE", namespace.to_string())
                     .stdout(process::Stdio::inherit())
                     .stderr(process::Stdio::inherit())
                     .stdin(process::Stdio::inherit())

@@ -113,7 +113,7 @@ pub fn run(profile: radicle::Profile) -> Result<(), Box<dyn std::error::Error +

                         // If our node is not running, we simply skip this step, as the
                         // refs will be announced eventually, when the node restarts.
                         if let Ok(conn) = profile.node() {
                             conn.announce_refs(&url.id)?;
                             conn.announce_refs(&url.repo)?;
                         }
                     }
                 }

modified radicle/src/storage/git.rs

@@ -966,18 +966,18 @@ mod tests {

             });
             opts.remote_callbacks(callbacks);
             // Register the `rad://` transport.
             transport::register().unwrap();
             // Register the `heartwood://` transport.
             transport::remote::register().unwrap();
             let target = git2::Repository::init_bare(target_path).unwrap();
             let stream = net::TcpStream::connect(addr).unwrap();
             let smart = transport::Smart::singleton();
             let smart = transport::remote::Smart::singleton();
             smart.insert(proj, Box::new(stream.try_clone().unwrap()));
             // Fetch with the `rad://` transport.
             // Fetch with the `heartwood://` transport.
             target
                 .remote_anonymous(&format!("rad://{}", proj))
                 .remote_anonymous(&format!("heartwood://{remote}/{proj}"))
                 .unwrap()
                 .fetch(
                     &["refs/namespaces/*:refs/namespaces/*"],

modified radicle/src/storage/git/transport.rs

@@ -1,133 +1,2 @@

 //! Git sub-transport used for fetching radicle data.
 //!
 //! To have control over the communication, and to allow git streams to be multiplexed over
 //! existing TCP connections, we implement the [`git2::transport::SmartSubtransport`] trait.
 //!
 //! We choose `rad` as the URL scheme for this custom transport, and include only the identity
 //! of the repository we're looking to fetch, eg. `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`, since
 //! we expect a connection to a host to already be established.
 //!
 //! We then maintain a map from identifier to stream, for all active streams, ie. streams that
 //! are associated with an underlying TCP connection. When a URL is requested, we lookup
 //! the stream and return it to the [`git2`] smart-protocol implementation, so that it can carry
 //! out the git smart protocol.
 //!
 //! This module is meant to be used by first registering our transport with [`register`] and then
 //! adding or removing streams through [`Smart`], which can be obtained via [`Smart::singleton`].
 mod url;
 use std::collections::HashMap;
 use std::str::FromStr;
 use std::sync::atomic;
 use std::sync::{Arc, Mutex};
 use git2::transport::SmartSubtransportStream;
 use once_cell::sync::Lazy;
 use crate::git;
 use crate::identity::Id;
 pub use url::{Url, UrlError};
 /// The map of git smart sub-transport streams. We keep a global map because we have
 /// no control over how [`git2::transport::register`] instantiates our [`Smart`] transport
 /// or its underlying streams.
 static STREAMS: Lazy<Arc<Mutex<HashMap<Id, Stream>>>> = Lazy::new(Default::default);
 /// The stream associated with a repository.
 type Stream = Box<dyn SmartSubtransportStream>;
 /// Git transport protocol over an I/O stream.
 #[derive(Clone)]
 pub struct Smart {
     /// The underlying active streams, keyed by repository identifier.
     streams: Arc<Mutex<HashMap<Id, Stream>>>,
 }
 impl Smart {
     /// Get access to the radicle smart transport protocol.
     /// The returned object has mutable access to the underlying stream map, and is safe to clone.
     pub fn singleton() -> Self {
         Self {
             streams: STREAMS.clone(),
         }
     }
     /// Take a stream from the map.
     /// This makes the stream unavailable until it is re-inserted.
     pub fn take(&self, id: &Id) -> Option<Stream> {
         #[allow(clippy::unwrap_used)]
         self.streams.lock().unwrap().remove(id)
     }
     pub fn insert(&self, id: Id, stream: Stream) {
         #[allow(clippy::unwrap_used)]
         self.streams.lock().unwrap().insert(id, stream);
     }
 }
 impl git2::transport::SmartSubtransport for Smart {
     /// Run a git service on this transport.
     ///
     /// Based on the URL, which must be of the form `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`,
     /// we retrieve an underlying stream and return it.
     ///
     /// We only support the upload-pack service, since only fetches are authorized by the
     /// remote.
     fn action(
         &self,
         url: &str,
         action: git2::transport::Service,
     ) -> Result<Box<dyn git2::transport::SmartSubtransportStream>, git2::Error> {
         let url = Url::from_str(url).map_err(|e| git2::Error::from_str(e.to_string().as_str()))?;
         if let Some(stream) = self.take(&url.id) {
             match action {
                 git2::transport::Service::UploadPackLs => {}
                 git2::transport::Service::UploadPack => {}
                 git2::transport::Service::ReceivePack => {
                     return Err(git2::Error::from_str(
                         "git-receive-pack is not supported with the custom transport",
                     ));
                 }
                 git2::transport::Service::ReceivePackLs => {
                     return Err(git2::Error::from_str(
                         "git-receive-pack is not supported with the custom transport",
                     ));
                 }
             }
             Ok(stream)
         } else {
             Err(git2::Error::from_str(&format!(
                 "repository {} does not have an associated stream",
                 url.id
             )))
         }
     }
     fn close(&self) -> Result<(), git2::Error> {
         Ok(())
     }
 }
 /// Register the radicle transport with `git`.
 ///
 /// Returns an error if called more than once.
 ///
 pub fn register() -> Result<(), git2::Error> {
     static REGISTERED: atomic::AtomicBool = atomic::AtomicBool::new(false);
     // Registration is not thread-safe, so make sure we prevent re-entrancy.
     if !REGISTERED.swap(true, atomic::Ordering::SeqCst) {
         unsafe {
             let prefix = git::url::Scheme::Radicle.to_string();
             git2::transport::register(&prefix, move |remote| {
                 git2::transport::Transport::smart(remote, false, Smart::singleton())
             })
         }
     } else {
         Err(git2::Error::from_str(
             "custom git transport is already registered",
         ))
     }
 }
 pub mod local;
 pub mod remote;

added radicle/src/storage/git/transport/local.rs

@@ -0,0 +1,2 @@

+	`pub mod url;`
+	`pub use url::{Url, UrlError};`

added radicle/src/storage/git/transport/local/url.rs

@@ -0,0 +1,106 @@

 //! Git local transport URLs.
 use std::str::FromStr;
 use thiserror::Error;
 use crate::{
     crypto,
     identity::{Id, IdError},
 };
 /// Repository namespace.
 type Namespace = crypto::PublicKey;
 #[derive(Debug, Error)]
 pub enum UrlError {
     /// Invalid format.
     #[error("invalid url format: expected `rad://<repo>[/<namespace>]`")]
     InvalidFormat,
     /// Unsupported URL scheme.
     #[error("unsupported scheme: expected `rad://`")]
     UnsupportedScheme,
     /// Invalid repository identifier.
     #[error("repo: {0}")]
     InvalidRepository(#[source] IdError),
     /// Invalid namespace.
     #[error("namespace: {0}")]
     InvalidNamespace(#[source] crypto::PublicKeyError),
 }
 /// A git local transport URL.
 ///
 /// * Used to content-address a repository, eg. when sharing projects.
 /// * Used as a remore url in a git working copy.
 ///
 /// `rad://<repo>[/<namespace>]`
 ///
 #[derive(Debug)]
 pub struct Url {
     /// Repository identifier.
     pub repo: Id,
     /// Repository sub-tree.
     pub namespace: Option<Namespace>,
 }
 impl Url {
     /// URL scheme.
     pub const SCHEME: &str = "rad";
 }
 impl FromStr for Url {
     type Err = UrlError;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let rest = s
             .strip_prefix("rad://")
             .ok_or(UrlError::UnsupportedScheme)?;
         let components = rest.split('/').collect::<Vec<_>>();
         let (resource, namespace) = match components.as_slice() {
             [resource] => (resource, None),
             [resource, namespace] => (resource, Some(namespace)),
             _ => return Err(UrlError::InvalidFormat),
         };
         let resource = Id::from_str(resource).map_err(UrlError::InvalidRepository)?;
         let namespace = namespace
             .map(|pk| Namespace::from_str(pk).map_err(UrlError::InvalidNamespace))
             .transpose()?;
         Ok(Url {
             repo: resource,
             namespace,
         })
     }
 }
 #[cfg(test)]
 mod test {
     use super::*;
     #[test]
     fn test_url_parse() {
         let repo = Id::from_str("z2w8RArM3gaBXZxXhQUswE3hhLcss").unwrap();
         let namespace =
             Namespace::from_str("z6Mkifeb5NPS6j7JP72kEQEeuqMTpCAVcHsJi1C86jGTzHRi").unwrap();
         let url = format!("rad://{repo}");
         let url = Url::from_str(&url).unwrap();
         assert_eq!(url.repo, repo);
         assert_eq!(url.namespace, None);
         let url = format!("rad://{repo}/{namespace}");
         let url = Url::from_str(&url).unwrap();
         assert_eq!(url.repo, repo);
         assert_eq!(url.namespace, Some(namespace));
         assert!(format!("heartwood://{repo}").parse::<Url>().is_err());
         assert!(format!("git://{repo}").parse::<Url>().is_err());
         assert!(format!("rad://{namespace}").parse::<Url>().is_err());
         assert!(format!("rad://{repo}/{namespace}/fnord")
             .parse::<Url>()
             .is_err());
     }
 }

added radicle/src/storage/git/transport/remote.rs

@@ -0,0 +1,131 @@

 //! Git sub-transport used for fetching radicle data.
 //!
 //! To have control over the communication, and to allow git streams to be multiplexed over
 //! existing TCP connections, we implement the [`git2::transport::SmartSubtransport`] trait.
 //!
 //! We choose `rad` as the URL scheme for this custom transport, and include only the identity
 //! of the repository we're looking to fetch, eg. `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`, since
 //! we expect a connection to a host to already be established.
 //!
 //! We then maintain a map from identifier to stream, for all active streams, ie. streams that
 //! are associated with an underlying TCP connection. When a URL is requested, we lookup
 //! the stream and return it to the [`git2`] smart-protocol implementation, so that it can carry
 //! out the git smart protocol.
 //!
 //! This module is meant to be used by first registering our transport with [`register`] and then
 //! adding or removing streams through [`Smart`], which can be obtained via [`Smart::singleton`].
 pub mod url;
 use std::collections::HashMap;
 use std::str::FromStr;
 use std::sync::atomic;
 use std::sync::{Arc, Mutex};
 use git2::transport::SmartSubtransportStream;
 use once_cell::sync::Lazy;
 use crate::identity::Id;
 pub use url::{Url, UrlError};
 /// The map of git smart sub-transport streams. We keep a global map because we have
 /// no control over how [`git2::transport::register`] instantiates our [`Smart`] transport
 /// or its underlying streams.
 static STREAMS: Lazy<Arc<Mutex<HashMap<Id, Stream>>>> = Lazy::new(Default::default);
 /// The stream associated with a repository.
 type Stream = Box<dyn SmartSubtransportStream>;
 /// Git transport protocol over an I/O stream.
 #[derive(Clone)]
 pub struct Smart {
     /// The underlying active streams, keyed by repository identifier.
     streams: Arc<Mutex<HashMap<Id, Stream>>>,
 }
 impl Smart {
     /// Get access to the radicle smart transport protocol.
     /// The returned object has mutable access to the underlying stream map, and is safe to clone.
     pub fn singleton() -> Self {
         Self {
             streams: STREAMS.clone(),
         }
     }
     /// Take a stream from the map.
     /// This makes the stream unavailable until it is re-inserted.
     pub fn take(&self, id: &Id) -> Option<Stream> {
         #[allow(clippy::unwrap_used)]
         self.streams.lock().unwrap().remove(id)
     }
     pub fn insert(&self, id: Id, stream: Stream) {
         #[allow(clippy::unwrap_used)]
         self.streams.lock().unwrap().insert(id, stream);
     }
 }
 impl git2::transport::SmartSubtransport for Smart {
     /// Run a git service on this transport.
     ///
     /// Based on the URL, which must be of the form `rad://zP1GztjSdYNHK7jpdrXbaJ6Ki2Ke`,
     /// we retrieve an underlying stream and return it.
     ///
     /// We only support the upload-pack service, since only fetches are authorized by the
     /// remote.
     fn action(
         &self,
         url: &str,
         action: git2::transport::Service,
     ) -> Result<Box<dyn git2::transport::SmartSubtransportStream>, git2::Error> {
         let url = Url::from_str(url).map_err(|e| git2::Error::from_str(e.to_string().as_str()))?;
         if let Some(stream) = self.take(&url.repo) {
             match action {
                 git2::transport::Service::UploadPackLs => {}
                 git2::transport::Service::UploadPack => {}
                 git2::transport::Service::ReceivePack => {
                     return Err(git2::Error::from_str(
                         "git-receive-pack is not supported with the custom transport",
                     ));
                 }
                 git2::transport::Service::ReceivePackLs => {
                     return Err(git2::Error::from_str(
                         "git-receive-pack is not supported with the custom transport",
                     ));
                 }
             }
             Ok(stream)
         } else {
             Err(git2::Error::from_str(&format!(
                 "repository {} does not have an associated stream",
                 url.repo
             )))
         }
     }
     fn close(&self) -> Result<(), git2::Error> {
         Ok(())
     }
 }
 /// Register the radicle transport with `git`.
 ///
 /// Returns an error if called more than once.
 ///
 pub fn register() -> Result<(), git2::Error> {
     static REGISTERED: atomic::AtomicBool = atomic::AtomicBool::new(false);
     // Registration is not thread-safe, so make sure we prevent re-entrancy.
     if !REGISTERED.swap(true, atomic::Ordering::SeqCst) {
         unsafe {
             git2::transport::register(Url::SCHEME, move |remote| {
                 git2::transport::Transport::smart(remote, false, Smart::singleton())
             })
         }
     } else {
         Err(git2::Error::from_str(
             "custom git transport is already registered",
         ))
     }
 }

added radicle/src/storage/git/transport/remote/url.rs

@@ -0,0 +1,115 @@

 //! Git remote transport URLs.
 use std::str::FromStr;
 use thiserror::Error;
 use crate::{
     crypto,
     identity::{Id, IdError},
 };
 type NodeId = crypto::PublicKey;
 type Namespace = crypto::PublicKey;
 #[derive(Debug, Error)]
 pub enum UrlError {
     /// Invalid format.
     #[error("invalid url format: expected `heartwood://<node>/<repo>[/<namespace>]`")]
     InvalidFormat,
     /// Unsupported URL scheme.
     #[error("unsupported scheme: expected `heartwood://`")]
     UnsupportedScheme,
     /// Invalid node identifier.
     #[error("node: {0}")]
     InvalidNode(#[source] crypto::PublicKeyError),
     /// Invalid repository identifier.
     #[error("repo: {0}")]
     InvalidRepository(#[source] IdError),
     /// Invalid namespace.
     #[error("namespace: {0}")]
     InvalidNamespace(#[source] crypto::PublicKeyError),
 }
 /// A git remote transport URL.
 ///
 /// `heartwood://<node>/<repo>[/<namespace>]`
 ///
 #[derive(Debug)]
 pub struct Url {
     /// Node identifier.
     pub node: NodeId,
     /// Repository identifier.
     pub repo: Id,
     /// Repository sub-tree.
     pub namespace: Option<Namespace>,
 }
 impl Url {
     /// URL scheme.
     pub const SCHEME: &str = "heartwood";
 }
 impl FromStr for Url {
     type Err = UrlError;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let rest = s
             .strip_prefix("heartwood://")
             .ok_or(UrlError::UnsupportedScheme)?;
         let components = rest.split('/').collect::<Vec<_>>();
         let (node, resource, namespace) = match components.as_slice() {
             [node, resource] => (node, resource, None),
             [node, resource, namespace] => (node, resource, Some(namespace)),
             _ => return Err(UrlError::InvalidFormat),
         };
         let node = NodeId::from_str(node).map_err(UrlError::InvalidNode)?;
         let resource = Id::from_str(resource).map_err(UrlError::InvalidRepository)?;
         let namespace = namespace
             .map(|pk| Namespace::from_str(pk).map_err(UrlError::InvalidNamespace))
             .transpose()?;
         Ok(Url {
             node,
             repo: resource,
             namespace,
         })
     }
 }
 #[cfg(test)]
 mod test {
     use super::*;
     #[test]
     fn test_url_parse() {
         let node = NodeId::from_str("z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK").unwrap();
         let repo = Id::from_str("z2w8RArM3gaBXZxXhQUswE3hhLcss").unwrap();
         let namespace =
             Namespace::from_str("z6Mkifeb5NPS6j7JP72kEQEeuqMTpCAVcHsJi1C86jGTzHRi").unwrap();
         let url = format!("heartwood://{node}/{repo}");
         let url = Url::from_str(&url).unwrap();
         assert_eq!(url.node, node);
         assert_eq!(url.repo, repo);
         assert_eq!(url.namespace, None);
         let url = format!("heartwood://{node}/{repo}/{namespace}");
         let url = Url::from_str(&url).unwrap();
         assert_eq!(url.node, node);
         assert_eq!(url.repo, repo);
         assert_eq!(url.namespace, Some(namespace));
         assert!(format!("heartwood://{node}").parse::<Url>().is_err());
         assert!(format!("rad://{node}").parse::<Url>().is_err());
         assert!(format!("heartwood://{node}/{namespace}")
             .parse::<Url>()
             .is_err());
         assert!(format!("heartwood://{node}/{repo}/{namespace}/fnord")
             .parse::<Url>()
             .is_err());
     }
 }

deleted radicle/src/storage/git/transport/url.rs

@@ -1,79 +0,0 @@

 //! Git transport URLs.
 use std::str::FromStr;
 use thiserror::Error;
 use crate::crypto::PublicKey;
 use crate::{crypto, git, identity};
 #[derive(Debug, Error)]
 pub enum UrlError {
     /// Failed to parse.
     #[error(transparent)]
     Parse(#[from] git::url::parse::Error),
     /// Unsupported URL scheme.
     #[error("{0}: unsupported scheme: expected `rad://`")]
     UnsupportedScheme(git::Url),
     /// Missing host.
     #[error("{0}: missing id")]
     MissingId(git::Url),
     /// Invalid remote repository identifier.
     #[error("{0}: id: {1}")]
     InvalidId(git::Url, identity::IdError),
     /// Invalid public key.
     #[error("{0}: key: {1}")]
     InvalidKey(git::Url, crypto::PublicKeyError),
 }
 /// A git remote URL.
 ///
 /// `rad://<id>/[<pubkey>]`
 ///
 /// Eg. `rad://zUBDc1UdoEzbpaGcNXqauQkERJ8r` without the public key,
 /// and `rad://zUBDc1UdoEzbpaGcNXqauQkERJ8r/zCQTxdZGCzQXWBV3XbY3fgkHM3gfkLGyYMd2nL5R2MxQv` with.
 ///
 #[derive(Debug)]
 pub struct Url {
     pub id: identity::Id,
     pub public_key: Option<PublicKey>,
 }
 impl FromStr for Url {
     type Err = UrlError;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let url: git::Url = s.as_bytes().try_into()?;
         Url::try_from(url)
     }
 }
 impl TryFrom<git::Url> for Url {
     type Error = UrlError;
     fn try_from(url: git::Url) -> Result<Self, Self::Error> {
         if url.scheme != git::url::Scheme::Radicle {
             return Err(Self::Error::UnsupportedScheme(url));
         }
         let id: identity::Id = url
             .host
             .as_ref()
             .ok_or_else(|| Self::Error::MissingId(url.clone()))?
             .parse()
             .map_err(|e| Self::Error::InvalidId(url.clone(), e))?;
         let public_key: Option<PublicKey> = if url.path.is_empty() {
             Ok(None)
         } else {
             let path = url.path.to_string();
             path.strip_prefix('/')
                 .unwrap_or(&path)
                 .parse()
                 .map(Some)
                 .map_err(|e| Self::Error::InvalidKey(url.clone(), e))
         }?;
         Ok(Url { id, public_key })
     }
 }