27f3951 node: Implement private repos — heartwood

modified radicle-cli/src/commands/init.rs

@@ -9,6 +9,7 @@ use serde_json as json;

 use radicle::crypto::ssh;
 use radicle::git::RefString;
 use radicle::identity::Visibility;
 use radicle::node::tracking::Scope;
 use radicle::node::{Handle, NodeId};
 use radicle::profile;

@@ -172,6 +173,7 @@ pub fn init(options: Options, profile: &profile::Profile) -> anyhow::Result<()>

     let path = options.path.unwrap_or_else(|| cwd.clone());
     let path = path.as_path().canonicalize()?;
     let interactive = options.interactive;
     let visibility = Visibility::default();
     term::headline(format!(
         "Initializing radicle 👾 project in {}",

@@ -222,6 +224,7 @@ pub fn init(options: Options, profile: &profile::Profile) -> anyhow::Result<()>

         &name,
         &description,
         branch,
         visibility,
         &signer,
         &profile.storage,
     ) {

modified radicle-cli/src/commands/ls.rs

@@ -1,7 +1,5 @@

 use std::ffi::OsString;
 use radicle::storage::{ReadRepository, ReadStorage};
 use crate::terminal as term;
 use crate::terminal::args::{Args, Error, Help};

@@ -53,26 +51,8 @@ pub fn run(options: Options, ctx: impl term::Context) -> anyhow::Result<()> {

     let storage = &profile.storage;
     let mut table = term::Table::default();
     for id in storage.repositories()? {
         let repo = match storage.repository(id) {
             Ok(repo) => repo,
             Err(err) => {
                 if options.verbose {
                     term::warning(&format!("failed to load project '{id}': {err}"));
                 }
                 continue;
             }
         };
         let head = match repo.head() {
             Ok((_, head)) => head,
             Err(err) => {
                 if options.verbose {
                     term::warning(&format!("failed to get head of project '{id}': {err}"));
                 }
                 continue;
             }
         };
         let proj = match repo.project() {
     for (id, head, doc) in storage.repositories()? {
         let proj = match doc.verified()?.project() {
             Ok(proj) => proj,
             Err(err) => {
                 if options.verbose {

modified radicle-httpd/src/api/v1/delegates.rs

@@ -37,7 +37,7 @@ async fn delegates_projects_handler(

     let storage = &ctx.profile.storage;
     let routing = &ctx.profile.routing()?;
     let projects = storage
         .repositories()?
         .inventory()?
         .into_iter()
         .filter_map(|id| {
             let Ok(repo) = storage.repository(id) else { return None };

modified radicle-httpd/src/api/v1/projects.rs

@@ -81,7 +81,7 @@ async fn project_root_handler(

     let storage = &ctx.profile.storage;
     let routing = &ctx.profile.routing()?;
     let projects = storage
         .repositories()?
         .inventory()?
         .into_iter()
         .filter_map(|id| {
             let Ok(repo) = storage.repository(id) else { return None };

modified radicle-httpd/src/api/v1/stats.rs

@@ -2,6 +2,7 @@ use axum::extract::State;

 use axum::response::IntoResponse;
 use axum::routing::get;
 use axum::{Json, Router};
 use radicle::storage::ReadStorage as _;
 use serde_json::json;
 use crate::api::error::Error;

@@ -17,7 +18,7 @@ pub fn router(ctx: Context) -> Router {

 /// `GET /stats`
 async fn stats_handler(State(ctx): State<Context>) -> impl IntoResponse {
     let storage = &ctx.profile.storage;
     let projects = storage.repositories()?.len();
     let projects = storage.inventory()?.len();
     Ok::<_, Error>(Json(
         json!({ "projects": { "count": projects }, "users": { "count": 0 } }),

modified radicle-httpd/src/test.rs

@@ -16,6 +16,7 @@ use radicle::crypto::ssh::keystore::MemorySigner;

 use radicle::crypto::ssh::Keystore;
 use radicle::crypto::{KeyPair, Seed, Signer};
 use radicle::git::{raw as git2, RefString};
 use radicle::identity::Visibility;
 use radicle::node;
 use radicle::node::address as AddressStore;
 use radicle::node::routing as RoutingStore;

@@ -175,8 +176,17 @@ fn seed_with_signer<G: Signer>(dir: &Path, profile: radicle::Profile, signer: &G

     let name = "hello-world".to_string();
     let description = "Rad repository for tests".to_string();
     let branch = RefString::try_from("master").unwrap();
     let (id, _, _) =
         radicle::rad::init(&repo, &name, &description, branch, signer, &profile.storage).unwrap();
     let visibility = Visibility::default();
     let (id, _, _) = radicle::rad::init(
         &repo,
         &name,
         &description,
         branch,
         visibility,
         signer,
         &profile.storage,
     )
     .unwrap();
     let storage = &profile.storage;
     let repo = storage.repository(id).unwrap();

modified radicle-node/src/service.rs

@@ -20,6 +20,7 @@ use localtime::{LocalDuration, LocalTime};

 use log::*;
 use nonempty::NonEmpty;
 use radicle::identity;
 use radicle::node::address;
 use radicle::node::address::{AddressBook, KnownAddress};
 use radicle::node::config::PeerConfig;

@@ -109,9 +110,13 @@ pub enum Error {

     #[error(transparent)]
     Storage(#[from] storage::Error),
     #[error(transparent)]
     Refs(#[from] storage::refs::Error),
     #[error(transparent)]
     Routing(#[from] routing::Error),
     #[error(transparent)]
     Tracking(#[from] tracking::Error),
     #[error(transparent)]
     Identity(#[from] identity::IdentityError),
     #[error("namespaces error: {0}")]
     Namespaces(#[from] NamespacesError),
 }

@@ -590,7 +595,8 @@ where

         }
     }
     pub fn fetch(&mut self, rid: Id, from: &NodeId) {
     /// Initiate an outgoing fetch for some repository.
     fn fetch(&mut self, rid: Id, from: &NodeId) {
         let Some(session) = self.sessions.get_mut(from) else {
             error!(target: "service", "Session {from} does not exist; cannot initiate fetch");
             return;

@@ -716,6 +722,21 @@ where

         }
     }
     /// Called when a remote requests a repository be uploaded to it.
     /// The upload is authorized if this function returns `true`.
     pub fn upload(&mut self, rid: Id, remote: &NodeId) -> bool {
         let Ok(repo) = self.storage.repository(rid) else {
             return false;
         };
         let Ok((_, doc)) = repo.identity_doc() else {
             return false;
         };
         if !doc.is_visible_to(remote) {
             return false;
         }
         true
     }
     /// Inbound connection attempt.
     pub fn accepted(&mut self, addr: Address) -> bool {
         // Always accept trusted connections.

@@ -1266,8 +1287,9 @@ where

         &mut self,
         rid: Id,
         remotes: impl IntoIterator<Item = NodeId>,
     ) -> Result<(), storage::Error> {
     ) -> Result<(), Error> {
         let repo = self.storage.repository(rid)?;
         let (_, doc) = repo.identity_doc()?;
         let peers = self.sessions.connected().map(|(_, p)| p);
         let timestamp = self.time();
         let mut refs = BoundedVec::<_, REF_REMOTE_LIMIT>::new();

@@ -1293,7 +1315,13 @@ where

         });
         let ann = msg.signed(&self.signer);
         self.outbox.broadcast(ann, peers);
         self.outbox.broadcast(
             ann,
             peers.filter(|p| {
                 // Only announce to peers who are allowed to view this repo.
                 doc.is_visible_to(&p.id)
             }),
         );
         Ok(())
     }

modified radicle-node/src/test/environment.rs

@@ -17,7 +17,7 @@ use radicle::crypto::test::signer::MockSigner;

 use radicle::crypto::{KeyPair, Seed, Signer};
 use radicle::git;
 use radicle::git::refname;
 use radicle::identity::Id;
 use radicle::identity::{Id, Visibility};
 use radicle::node::address::Book;
 use radicle::node::routing;
 use radicle::node::routing::Store;

@@ -402,6 +402,7 @@ impl<G: cyphernet::Ecdh<Pk = NodeId> + Signer + Clone> Node<G> {

             name,
             description,
             refname!("master"),
             Visibility::default(),
             &self.signer,
             &self.storage,
         )

modified radicle-node/src/test/peer.rs

@@ -6,6 +6,7 @@ use std::str::FromStr;

 use log::*;
 use radicle::identity::Visibility;
 use radicle::node::address::Store;
 use radicle::node::{address, Alias, ConnectOptions};
 use radicle::rad;

@@ -132,6 +133,7 @@ impl<G: Signer> Peer<Storage, G> {

             name,
             description,
             radicle::git::refname!("master"),
             Visibility::default(),
             self.signer(),
             self.storage(),
         )

modified radicle-node/src/tests.rs

@@ -8,6 +8,7 @@ use std::time;

 use crossbeam_channel as chan;
 use netservices::Direction as Link;
 use radicle::identity::Visibility;
 use radicle::node::routing::Store as _;
 use radicle::node::ConnectOptions;
 use radicle::storage::ReadRepository;

@@ -1282,6 +1283,7 @@ fn test_push_and_pull() {

         "alice",
         "alice's repo",
         git::refname!("master"),
         Visibility::default(),
         alice.signer(),
         alice.storage(),
     )

modified radicle-node/src/worker.rs

@@ -11,7 +11,7 @@ use crossbeam_channel as chan;

 use radicle::identity::Id;
 use radicle::prelude::NodeId;
 use radicle::storage::{Namespaces, ReadRepository, RefUpdate};
 use radicle::storage::{Namespaces, ReadRepository, ReadStorage, RefUpdate};
 use radicle::{git, storage, Storage};
 use crate::runtime::{thread, Handle};

@@ -66,6 +66,12 @@ pub enum UploadError {

     PacketLine(io::Error),
     #[error(transparent)]
     Io(#[from] io::Error),
     #[error("{0} is not authorized to fetch {1}")]
     Unauthorized(NodeId, Id),
     #[error(transparent)]
     Storage(#[from] radicle::storage::Error),
     #[error(transparent)]
     Identity(#[from] radicle::identity::IdentityError),
 }
 impl UploadError {

@@ -309,6 +315,13 @@ impl Worker {

         };
         log::debug!(target: "worker", "Received Git request pktline for {rid}..");
         let repo = self.storage.repository(rid)?;
         let (_, doc) = repo.identity_doc()?;
         if !doc.is_visible_to(&remote) {
             return Err(UploadError::Unauthorized(remote, rid));
         }
         match self._upload_pack(rid, remote, request, stream, stream_r, stream_w) {
             Ok(()) => {
                 log::debug!(target: "worker", "Upload of {rid} to {remote} on stream {stream} exited successfully");

modified radicle-tools/src/rad-init.rs

@@ -1,6 +1,6 @@

 use std::path::Path;
 use radicle::{git, Profile};
 use radicle::{git, identity::Visibility, Profile};
 fn main() -> anyhow::Result<()> {
     let cwd = Path::new(".").canonicalize()?;

@@ -13,6 +13,7 @@ fn main() -> anyhow::Result<()> {

         &name,
         "",
         git::refname!("master"),
         Visibility::default(),
         &signer,
         &profile.storage,
     )?;

modified radicle/src/identity.rs

@@ -15,7 +15,7 @@ use crate::storage::{refs, ReadRepository, RemoteId};

 pub use crypto::PublicKey;
 pub use did::Did;
 pub use doc::{Doc, Id, IdError, PayloadError};
 pub use doc::{Doc, Id, IdError, PayloadError, Visibility};
 pub use project::Project;
 /// Untrusted, well-formed input.

@@ -164,6 +164,7 @@ impl Identity<Untrusted> {

         })
     }
 }
 #[cfg(test)]
 mod test {
     use qcheck_macros::quickcheck;

modified radicle/src/identity/doc.rs

@@ -135,6 +135,26 @@ impl Deref for DocAt {

     }
 }
 /// Repository visibility.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase", tag = "type")]
 pub enum Visibility {
     /// Anyone and everyone.
     #[default]
     Public,
     /// Delegates plus the allowed DIDs.
     Private {
         #[serde(default, skip_serializing_if = "Vec::is_empty")]
         allow: Vec<Did>,
     },
 }
 impl Visibility {
     pub fn is_public(&self) -> bool {
         matches!(self, Self::Public)
     }
 }
 /// An identity document.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]

@@ -145,12 +165,25 @@ pub struct Doc<V> {

     pub delegates: NonEmpty<Did>,
     /// The signature threshold.
     pub threshold: usize,
     /// Repository visibility.
     #[serde(default, skip_serializing_if = "Visibility::is_public")]
     pub visibility: Visibility,
     #[serde(skip)]
     verified: PhantomData<V>,
 }
 impl<V> Doc<V> {
     /// Check whether this document and the associated repository is visible to the given peer.
     pub fn is_visible_to(&self, peer: &PublicKey) -> bool {
         match &self.visibility {
             Visibility::Public => true,
             Visibility::Private { allow } => {
                 allow.contains(&Did::from(*peer)) || self.is_delegate(peer)
             }
         }
     }
     pub fn canonical_head(repo: &storage::git::Repository) -> Result<Oid, DocError> {
         repo.backend
             .refname_to_id(storage::git::CANONICAL_IDENTITY.as_str())

@@ -327,17 +360,23 @@ impl Doc<Verified> {

             payload: self.payload,
             delegates: self.delegates,
             threshold: self.threshold,
             visibility: self.visibility,
             verified: PhantomData,
         }
     }
 }
 impl Doc<Unverified> {
     pub fn initial(project: Project, delegate: Did) -> Self {
         Self::new(project, NonEmpty::new(delegate), 1)
     pub fn initial(project: Project, delegate: Did, visibility: Visibility) -> Self {
         Self::new(project, NonEmpty::new(delegate), 1, visibility)
     }
     pub fn new(project: Project, delegates: NonEmpty<Did>, threshold: usize) -> Self {
     pub fn new(
         project: Project,
         delegates: NonEmpty<Did>,
         threshold: usize,
         visibility: Visibility,
     ) -> Self {
         let project =
             serde_json::to_value(project).expect("Doc::initial: payload must be serializable");

@@ -345,6 +384,7 @@ impl Doc<Unverified> {

             payload: BTreeMap::from_iter([(PayloadId::project(), Payload::from(project))]),
             delegates,
             threshold,
             visibility,
             verified: PhantomData,
         }
     }

@@ -377,6 +417,7 @@ impl Doc<Unverified> {

             payload: self.payload,
             delegates: self.delegates,
             threshold: self.threshold,
             visibility: self.visibility,
             verified: PhantomData,
         })
     }

@@ -424,6 +465,7 @@ mod test {

             "heartwood",
             "Radicle Heartwood Protocol & Stack",
             git::refname!("master"),
             Visibility::default(),
             &delegate,
             &storage,
         )

@@ -470,6 +512,7 @@ mod test {

             "heartwood",
             "Radicle Heartwood Protocol & Stack",
             git::refname!("master"),
             Visibility::default(),
             &delegate,
             &storage,
         )

@@ -484,4 +527,28 @@ mod test {

         let (_, bytes) = doc.encode().unwrap();
         assert_eq!(Doc::from_json(&bytes).unwrap().verified().unwrap(), doc);
     }
     #[test]
     fn test_visibility_json() {
         use std::str::FromStr;
         assert_eq!(
             serde_json::to_value(Visibility::Public).unwrap(),
             serde_json::json!({ "type": "public" })
         );
         assert_eq!(
             serde_json::to_value(Visibility::Private { allow: vec![] }).unwrap(),
             serde_json::json!({ "type": "private" })
         );
         assert_eq!(
             serde_json::to_value(Visibility::Private {
                 allow: vec![Did::from_str(
                     "did:key:z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT"
                 )
                 .unwrap()]
             })
             .unwrap(),
             serde_json::json!({ "type": "private", "allow": ["did:key:z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT"] })
         );
     }
 }

modified radicle/src/node/tracking/schema.sql

@@ -22,7 +22,6 @@ create table if not exists "repo-policies" (

   -- Valid values are:
   --
   -- "trusted"         track repository delegates and remotes in the `node-policies` table.
   -- "delegates-only"  only track repository delegates.
   -- "all"             track all remotes.
   --
   "scope"              text      default 'trusted',

modified radicle/src/rad.rs

@@ -9,7 +9,7 @@ use thiserror::Error;

 use crate::cob::ObjectId;
 use crate::crypto::{Signer, Verified};
 use crate::git;
 use crate::identity::doc::{DocError, Id};
 use crate::identity::doc::{DocError, Id, Visibility};
 use crate::identity::project::Project;
 use crate::identity::{doc, IdentityError};
 use crate::storage::git::transport;

@@ -54,6 +54,7 @@ pub fn init<G: Signer, S: WriteStorage>(

     name: &str,
     description: &str,
     default_branch: BranchName,
     visibility: Visibility,
     signer: &G,
     storage: S,
 ) -> Result<(Id, identity::Doc<Verified>, SignedRefs<Verified>), InitError> {

@@ -73,7 +74,7 @@ pub fn init<G: Signer, S: WriteStorage>(

                 .join(", "),
         )
     })?;
     let doc = identity::Doc::initial(proj, delegate).verified()?;
     let doc = identity::Doc::initial(proj, delegate, visibility).verified()?;
     let (project, _) = Repository::init(&doc, pk, storage, signer)?;
     let url = git::Url::from(project.id);

@@ -388,6 +389,7 @@ mod tests {

             "acme",
             "Acme's repo",
             git::refname!("master"),
             Visibility::default(),
             &signer,
             &storage,
         )

@@ -442,6 +444,7 @@ mod tests {

             "acme",
             "Acme's repo",
             git::refname!("master"),
             Visibility::default(),
             &alice,
             &storage,
         )

@@ -477,6 +480,7 @@ mod tests {

             "acme",
             "Acme's repo",
             git::refname!("master"),
             Visibility::default(),
             &signer,
             &storage,
         )

modified radicle/src/storage.rs

@@ -303,6 +303,7 @@ pub trait ReadStorage {

     /// Check whether storage contains a repository.
     fn contains(&self, rid: &Id) -> Result<bool, IdentityError>;
     /// Get the inventory of repositories hosted under this storage.
     /// This function should typically only return public repositories.
     fn inventory(&self) -> Result<Inventory, Error>;
     /// Open or create a read-only repository.
     fn repository(&self, rid: Id) -> Result<Self::Repository, Error>;

modified radicle/src/storage/git.rs

@@ -105,7 +105,13 @@ impl ReadStorage for Storage {

     }
     fn inventory(&self) -> Result<Inventory, Error> {
         self.repositories()
         let repos = self.repositories()?;
         Ok(repos
             .into_iter()
             .filter(|(_, _, doc)| doc.visibility.is_public())
             .map(|(rid, _, _)| rid)
             .collect())
     }
     fn repository(&self, rid: Id) -> Result<Self::Repository, Error> {

@@ -143,7 +149,7 @@ impl Storage {

         self.path.as_path()
     }
     pub fn repositories(&self) -> Result<Vec<Id>, Error> {
     pub fn repositories(&self) -> Result<Vec<(Id, Oid, Doc<Unverified>)>, Error> {
         let mut repos = Vec::new();
         for result in fs::read_dir(&self.path)? {

@@ -159,28 +165,45 @@ impl Storage {

             }
             let rid =
                 Id::try_from(path.file_name()).map_err(|_| Error::InvalidId(path.file_name()))?;
             let repo = self.repository(rid)?;
             let repo = match self.repository(rid) {
                 Ok(repo) => repo,
                 Err(e) => {
                     log::warn!(target: "storage", "Repository {rid} is invalid: {e}");
                     continue;
                 }
             };
             let doc = match repo.identity_doc() {
                 Ok((_, doc)) => doc,
                 Err(e) => {
                     log::warn!(target: "storage", "Repository {rid} is invalid: looking up doc: {e}");
                     continue;
                 }
             };
             // For performance reasons, we don't do a full repository check here.
             if let Err(e) = repo.head() {
                 log::warn!(target: "storage", "Repository {rid} is invalid: looking up head: {e}");
                 continue;
             }
             repos.push(rid);
             let head = match repo.head() {
                 Ok((_, head)) => head,
                 Err(e) => {
                     log::warn!(target: "storage", "Repository {rid} is invalid: looking up head: {e}");
                     continue;
                 }
             };
             repos.push((rid, head, doc));
         }
         Ok(repos)
     }
     pub fn inspect(&self) -> Result<(), Error> {
         for proj in self.repositories()? {
             let repo = self.repository(proj)?;
         for (rid, _, _) in self.repositories()? {
             let repo = self.repository(rid)?;
             for r in repo.raw().references()? {
                 let r = r?;
                 let name = r.name().ok_or(Error::InvalidRef)?;
                 let oid = r.target().ok_or(Error::InvalidRef)?;
                 println!("{} {oid} {name}", proj.urn());
                 println!("{} {oid} {name}", rid.urn());
             }
         }
         Ok(())

modified radicle/src/test/arbitrary.rs

@@ -10,6 +10,7 @@ use nonempty::NonEmpty;

 use qcheck::Arbitrary;
 use crate::collections::RandomMap;
 use crate::identity::doc::Visibility;
 use crate::identity::{
     doc::{Doc, Id},
     project::Project,

@@ -115,12 +116,25 @@ impl Arbitrary for Project {

     }
 }
 impl Arbitrary for Visibility {
     fn arbitrary(g: &mut qcheck::Gen) -> Self {
         if bool::arbitrary(g) {
             Visibility::Public
         } else {
             Visibility::Private {
                 allow: Vec::arbitrary(g),
             }
         }
     }
 }
 impl Arbitrary for Doc<Unverified> {
     fn arbitrary(g: &mut qcheck::Gen) -> Self {
         let proj = Project::arbitrary(g);
         let delegate = Did::arbitrary(g);
         let visibility = Visibility::arbitrary(g);
         Self::initial(proj, delegate)
         Self::initial(proj, delegate, visibility)
     }
 }

@@ -134,7 +148,8 @@ impl Arbitrary for Doc<Verified> {

             .try_into()
             .unwrap();
         let threshold = delegates.len() / 2 + 1;
         let doc: Doc<Unverified> = Doc::new(project, delegates, threshold);
         let visibility = Visibility::arbitrary(g);
         let doc: Doc<Unverified> = Doc::new(project, delegates, threshold, visibility);
         doc.verified().unwrap()
     }

modified radicle/src/test/fixtures.rs

@@ -2,6 +2,7 @@ use std::path::Path;

 use crate::crypto::{Signer, Verified};
 use crate::git;
 use crate::identity::doc::Visibility;
 use crate::identity::Id;
 use crate::rad;
 use crate::storage::git::transport;

@@ -25,7 +26,15 @@ pub fn storage<P: AsRef<Path>, G: Signer>(path: P, signer: &G) -> Result<Storage

         ("rx", "A pixel editor"),
     ] {
         let (repo, _) = repository(path.join("workdir").join(name));
         rad::init(&repo, name, desc, git::refname!("master"), signer, &storage)?;
         rad::init(
             &repo,
             name,
             desc,
             git::refname!("master"),
             Visibility::default(),
             signer,
             &storage,
         )?;
     }
     Ok(storage)

@@ -45,6 +54,7 @@ pub fn project<P: AsRef<Path>, G: Signer>(

         "acme",
         "Acme's repository",
         git::refname!("master"),
         Visibility::default(),
         signer,
         storage,
     )?;