cli: Improvements to sync & clone UX — heartwood

added radicle-cli/examples/rad-clone-partial-fail.md

@@ -0,0 +1,32 @@

 Eve knows about three seeds.
 ```
 $ rad node routing
 ╭─────────────────────────────────────────────────────╮
 │ RID                                 NID             │
 ├─────────────────────────────────────────────────────┤
 │ rad:z42hL2jL4XNk6K8oHQaSWfMgCL7ji   z6MknSL…StBU8Vi │
 │ rad:z42hL2jL4XNk6K8oHQaSWfMgCL7ji   z6MksFq…bS9wzpT │
 │ rad:z42hL2jL4XNk6K8oHQaSWfMgCL7ji   z6Mkt67…v4N1tRk │
 ╰─────────────────────────────────────────────────────╯
 ```
 When she tries to clone, one of those will fail to fetch. But the clone command
 still returns successfully.
 ```
 $ rad clone rad:z42hL2jL4XNk6K8oHQaSWfMgCL7ji --timeout 3
 ✓ Seeding policy updated for rad:z42hL2jL4XNk6K8oHQaSWfMgCL7ji with scope 'all'
 ✗ Fetching rad:z42hL2jL4XNk6K8oHQaSWfMgCL7ji from z6Mkt67…v4N1tRk.. error: failed to perform fetch handshake
 ✓ Fetching rad:z42hL2jL4XNk6K8oHQaSWfMgCL7ji from z6MknSL…StBU8Vi..
 ✗ Connecting to z6MksFq…bS9wzpT@[..].. error: connection reset
 ✓ Creating checkout in ./heartwood..
 ✓ Remote alice@z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi added
 ✓ Remote-tracking branch alice@z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi/master created for z6MknSL…StBU8Vi
 ✓ Repository successfully cloned under [..]/heartwood/
 ╭────────────────────────────────────╮
 │ heartwood                          │
 │ Radicle Heartwood Protocol & Stack │
 │ 0 issues · 0 patches               │
 ╰────────────────────────────────────╯
 Run `cd ./heartwood` to go to the repository directory.
 ```

modified radicle-cli/examples/rad-init-private-clone-seed.md

@@ -29,7 +29,7 @@ $ rad inspect --identity

 ``` ~bob
 $ rad ls --all --private
 $ rad clone rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --seed z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi --timeout 1
 $ rad clone rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --private --seed z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi --timeout 1
 ✓ Seeding policy updated for rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu with scope 'all'
 ✓ Fetching rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu from z6MknSL…StBU8Vi..
 ✓ Creating checkout in ./heartwood..

modified radicle-cli/examples/rad-init-private-clone.md

@@ -5,7 +5,7 @@ Bob tries to fetch it, and even though he's connected to Alice, it fails.

 $ rad ls
 ```
 ``` ~bob (fail)
 $ rad clone rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --seed z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi --timeout 1
 $ rad clone rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --private --seed z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi --timeout 1
 ✓ Seeding policy updated for rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu with scope 'all'
 ✗ Fetching rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu from z6MknSL…StBU8Vi.. error: failed to perform fetch handshake
 ✗ Error: repository rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu not found

@@ -21,7 +21,8 @@ $ rad sync --announce --timeout 3

 ✓ Synced with 1 node(s)
 ```
 Bob can now fetch the private repo:
 Bob can now fetch the private repo without specifying a seed, because he knows
 that Alice has the repo after she announced her refs:
 ``` ~bob
 $ rad sync rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --fetch

added radicle-cli/examples/rad-init-private-seed.md

@@ -0,0 +1,57 @@

 Alice allows Bob to fetch this repo, but doesn't announce it, which means
 that Bob needs to know to fetch it from Alice.
 ``` ~alice
 $ rad id update --title "Allow Bob" --description "" --allow did:key:z6Mkt67GdsW7715MEfRuP4pSZxJRJh6kj6Y48WRqVv4N1tRk -q
 [..]
 ```
 First, Bob seeds the repo.
 ``` ~bob
 $ rad seed rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --no-fetch
 [..]
 ```
 If Bob just tries to fetch it without specifying seeds, he gets an error:
 ``` ~bob
 $ rad sync rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --fetch
 ✗ Error: no seeds found for rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu
 ```
 If he specifies a seed that isn't in his routing table, he gets a warning and
 an error:
 ``` ~bob
 $ rad sync rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --fetch --seed z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi
 ! Warning: node z6MknSL…StBU8Vi is not connected or seeding.. skipping
 ✗ Error: no seeds found for rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu
 ```
 For this to work, Bob has to force the sync:
 ``` ~bob
 $ rad sync rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --fetch --seed z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi --force
 ✓ Fetching rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu from z6MknSL…StBU8Vi..
 ✓ Fetched repository from 1 seed(s)
 ```
 ``` ~bob
 $ rad ls --private --all
 ╭───────────────────────────────────────────────────────────────────────────────────────────────────────────╮
 │ Name        RID                                 Visibility   Head      Description                        │
 ├───────────────────────────────────────────────────────────────────────────────────────────────────────────┤
 │ heartwood   rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu   private      f2de534   radicle heartwood protocol & stack │
 ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
 Note that if multiple seeds are specified, the command succeeds as long as one
 seed succeeds.
 ``` ~bob
 $ rad sync rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu --fetch --seed z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi --seed z6MkwPUeUS2fJMfc2HZN1RQTQcTTuhw4HhPySB8JeUg2mVvx --force
 ✓ Fetching rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu from z6MknSL…StBU8Vi..
 ✗ Fetching rad:z2ug5mwNKZB8KGpBDRTrWHAMbvHCu from z6MkwPU…Ug2mVvx.. error: session does not exist; cannot initiate fetch
 ✓ Fetched repository from 1 seed(s)
 ```

modified radicle-cli/src/commands/clone.rs

@@ -41,13 +41,14 @@ Usage

     The `clone` command will use your local node's routing table to find seeds from
     which it can clone the repository.
     For private repositories, the `--seed` option can be passed to clone directly
     from a known seed in the privacy set.
     For private repositories, use the `--private` and `--seed` options, to clone directly
     from known seeds in the privacy set.
 Options
         --scope <scope>     Follow scope: `followed` or `all` (default: all)
     -s, --seed <nid>        Clone from this seed (may be specified multiple times)
         --private           Clone a private repository
         --timeout <secs>    Timeout for fetching repository (default: 9)
         --help              Print help

@@ -64,8 +65,6 @@ pub struct Options {

     scope: Scope,
     /// Sync settings.
     sync: SyncSettings,
     /// Fetch timeout.
     timeout: time::Duration,
 }
 impl Args for Options {

@@ -76,7 +75,6 @@ impl Args for Options {

         let mut id: Option<RepoId> = None;
         let mut scope = Scope::All;
         let mut sync = SyncSettings::default();
         let mut timeout = time::Duration::from_secs(9);
         let mut directory = None;
         while let Some(arg) = parser.next()? {

@@ -93,11 +91,14 @@ impl Args for Options {

                     scope = term::args::parse_value("scope", value)?;
                 }
                 Long("private") => {
                     sync.force = true;
                 }
                 Long("timeout") => {
                     let value = parser.value()?;
                     let secs = term::args::number(&value)?;
                     timeout = time::Duration::from_secs(secs as u64);
                     sync.timeout = time::Duration::from_secs(secs as u64);
                 }
                 Long("no-confirm") => {
                     // We keep this flag here for consistency though it doesn't have any effect,

@@ -129,7 +130,6 @@ impl Args for Options {

                 directory,
                 scope,
                 sync,
                 timeout,
             },
             vec![],
         ))

@@ -151,7 +151,7 @@ pub fn run(options: Options, ctx: impl term::Context) -> anyhow::Result<()> {

         options.id,
         options.directory.clone(),
         options.scope,
         options.sync.with_profile(&profile).timeout(options.timeout),
         options.sync.with_profile(&profile),
         &mut node,
         &signer,
         &profile.storage,

modified radicle-cli/src/commands/sync.rs

@@ -1,5 +1,6 @@

 use std::cmp::Ordering;
 use std::collections::BTreeSet;
 use std::collections::VecDeque;
 use std::ffi::OsString;
 use std::str::FromStr;
 use std::time;

@@ -42,6 +43,9 @@ Usage

     When `--fetch` is specified, any number of seeds may be given
     using the `--seed` option, eg. `--seed <nid>@<addr>:<port>`.
     To force a fetch even if there is no route to a seed (as is the case for
     private repositories), `--force` can be used.
     When `--replicas` is specified, the given replication factor will try
     to be matched. For example, `--replicas 5` will sync with 5 seeds.

@@ -61,6 +65,7 @@ Options

     -f, --fetch               Turn on fetching (default: true)
     -a, --announce            Turn on ref announcing (default: true)
     -i, --inventory           Turn on inventory announcing (default: false)
         --force               Force fetches from unknown seeds (default: false)
         --timeout   <secs>    How many seconds to wait while syncing
         --seed      <nid>     Sync with the given node (may be specified multiple times)
     -r, --replicas  <count>   Sync with a specific number of seeds

@@ -144,6 +149,7 @@ impl Args for Options {

         let mut fetch = false;
         let mut announce = false;
         let mut inventory = false;
         let mut force = false;
         let mut debug = false;
         let mut replicas = None;
         let mut seeds = BTreeSet::new();

@@ -161,6 +167,9 @@ impl Args for Options {

                 Long("fetch") | Short('f') => {
                     fetch = true;
                 }
                 Long("force") => {
                     force = true;
                 }
                 Long("replicas") | Short('r') => {
                     let val = parser.value()?;
                     let count = term::args::number(&val)?;

@@ -209,8 +218,10 @@ impl Args for Options {

             }
         }
         let sync = if inventory && (fetch || announce) {
             anyhow::bail!("`--inventory` cannot be used with `--fetch` or `--announce`");
         let sync = if inventory && (fetch || announce || force) {
             anyhow::bail!(
                 "`--inventory` cannot be used with `--fetch` or `--announce` or `--force`"
             );
         } else if inventory {
             SyncMode::Inventory
         } else {

@@ -219,12 +230,16 @@ impl Args for Options {

                 (true, false) => SyncDirection::Fetch,
                 (false, true) => SyncDirection::Announce,
             };
             if direction == SyncDirection::Announce && force {
                 anyhow::bail!("`--force` cannot be used without `--fetch`");
             }
             let settings = if seeds.is_empty() {
                 SyncSettings::from_replicas(replicas.unwrap_or(3))
             } else {
                 SyncSettings::from_seeds(seeds)
             }
             .timeout(timeout);
             .timeout(timeout)
             .force(force);
             SyncMode::Repo {
                 settings,

@@ -431,35 +446,46 @@ pub fn fetch(

     let local = node.nid()?;
     // Get seeds. This consults the local routing table only.
     let seeds = node.seeds(rid)?;
     // Target replicas, clamped by the maximum replicas possible.
     let replicas = settings
         .replicas
         .min(seeds.iter().filter(|s| s.nid != local).count());
     let sessions = node.sessions()?;
     // Target replicas, clamped by the maximum replicas possible,
     // unless `force` is true.
     let replicas = if settings.force {
         settings.replicas
     } else {
         settings
             .replicas
             .min(seeds.iter().filter(|s| s.nid != local).count())
     };
     let mut results = FetchResults::default();
     let (connected, mut disconnected) = seeds.partition();
     // Fetch from specified seeds, plus our preferred seeds.
     // Fetch from specified seeds.
     for nid in &settings.seeds {
         if !sessions.iter().any(|s| &s.nid == nid) {
             term::warning(format!("node {nid} is not connected.. skipping"));
         if !seeds.is_connected(nid) && !settings.force {
             term::warning(format!(
                 "node {} is not connected or seeding.. skipping",
                 term::format::node(nid)
             ));
             continue;
         }
         let result = fetch_from(rid, nid, settings.timeout, node)?;
         results.push(*nid, result);
     }
     if results.success().count() >= replicas {
         return Ok(results);
         if results.success().count() >= replicas {
             return Ok(results);
         }
     }
     // Fetch from connected seeds.
     let connected = connected
     let mut connected = connected
         .into_iter()
         .filter(|c| !results.contains(&c.nid))
         .map(|c| c.nid)
         .take(replicas)
         .collect::<Vec<_>>();
     for nid in connected {
         .collect::<VecDeque<_>>();
     while results.success().count() < replicas {
         let Some(nid) = connected.pop_front() else {
             break;
         };
         let result = fetch_from(rid, &nid, settings.timeout, node)?;
         results.push(nid, result);
     }

@@ -478,7 +504,7 @@ pub fn fetch(

             seed.addrs.into_iter().map(|ka| ka.addr),
             settings.timeout,
             node,
         )? {
         ) {
             let result = fetch_from(rid, &seed.nid, settings.timeout, node)?;
             results.push(seed.nid, result);
         }

@@ -492,7 +518,7 @@ fn connect(

     addrs: impl Iterator<Item = node::Address>,
     timeout: time::Duration,
     node: &mut Node,
 ) -> Result<bool, node::Error> {
 ) -> bool {
     // Try all addresses until one succeeds.
     for addr in addrs {
         let spinner = term::spinner(format!(

@@ -507,20 +533,24 @@ fn connect(

                 persistent: false,
                 timeout,
             },
         )?;
         );
         match cr {
             node::ConnectResult::Connected => {
             Ok(node::ConnectResult::Connected) => {
                 spinner.finish();
                 return Ok(true);
                 return true;
             }
             Ok(node::ConnectResult::Disconnected { reason }) => {
                 spinner.error(reason);
                 continue;
             }
             node::ConnectResult::Disconnected { .. } => {
                 spinner.failed();
             Err(e) => {
                 spinner.error(e);
                 continue;
             }
         }
     }
     Ok(false)
     false
 }
 fn fetch_from(

modified radicle-cli/src/node.rs

@@ -22,6 +22,9 @@ pub struct SyncSettings {

     pub replicas: usize,
     /// Sync with the given list of seeds.
     pub seeds: BTreeSet<NodeId>,
     /// Sync with the given seeds even if they aren't in our routing table.
     /// Can be used to fetch private repositories, for example.
     pub force: bool,
     /// How long to wait for syncing to complete.
     pub timeout: time::Duration,
 }

@@ -33,6 +36,7 @@ impl SyncSettings {

         Self {
             replicas: seeds.len(),
             seeds,
             force: false,
             timeout: DEFAULT_SYNC_TIMEOUT,
         }
     }

@@ -51,17 +55,22 @@ impl SyncSettings {

         self
     }
     /// Set the 'force' option.
     pub fn force(mut self, force: bool) -> Self {
         self.force = force;
         self
     }
     /// Use profile to populate sync settings, by adding preferred seeds if no seeds are specified,
     /// and removing the local node from the set.
     pub fn with_profile(mut self, profile: &Profile) -> Self {
         // If no seeds were specified, add up to `replica` seeds from the preferred seeds.
         // If no seeds were specified, add the preferred seeds.
         if self.seeds.is_empty() {
             self.seeds = profile
                 .config
                 .preferred_seeds
                 .iter()
                 .map(|p| p.id)
                 .take(self.replicas)
                 .collect();
         }
         // Remove our local node from the seed set just in case it was added by mistake.

@@ -75,6 +84,7 @@ impl Default for SyncSettings {

         Self {
             replicas: 3,
             seeds: BTreeSet::new(),
             force: false,
             timeout: DEFAULT_SYNC_TIMEOUT,
         }
     }

modified radicle-cli/tests/commands.rs

@@ -9,7 +9,7 @@ use radicle::node::config::seeds::{RADICLE_COMMUNITY_NODE, RADICLE_TEAM_NODE};

 use radicle::node::routing::Store as _;
 use radicle::node::Handle as _;
 use radicle::node::{Address, Alias, DEFAULT_TIMEOUT};
 use radicle::prelude::RepoId;
 use radicle::prelude::{NodeId, RepoId};
 use radicle::profile;
 use radicle::profile::Home;
 use radicle::storage::{ReadStorage, RefUpdate, RemoteRepository};

@@ -1191,6 +1191,63 @@ fn rad_clone_all() {

 }
 #[test]
 fn rad_clone_partial_fail() {
     let mut environment = Environment::new();
     let mut alice = environment.node(Config::test(Alias::new("alice")));
     let bob = environment.node(Config::test(Alias::new("bob")));
     let mut eve = environment.node(Config::test(Alias::new("eve")));
     let working = environment.tmp().join("working");
     let carol = NodeId::from_str("z6MksFqXN3Yhqk8pTJdUGLwBTkRfQvwZXPqR2qMEhbS9wzpT").unwrap();
     // Setup a test project.
     let acme = alice.project("heartwood", "Radicle Heartwood Protocol & Stack");
     let mut alice = alice.spawn();
     let mut bob = bob.spawn();
     // Make Even think she knows about a seed called "carol" that has the repo.
     eve.db
         .addresses_mut()
         .insert(
             &carol,
             node::Features::SEED,
             Alias::new("carol"),
 ,
             localtime::LocalTime::now().as_secs(),
             [node::KnownAddress::new(
                 // Eve will fail to connect to this address.
                 node::Address::from(net::SocketAddr::from(([0, 0, 0, 0], 19873))),
                 node::address::Source::Imported,
             )],
         )
         .unwrap();
     eve.db
         .routing_mut()
         .insert([&acme], carol, localtime::LocalTime::now().as_secs())
         .unwrap();
     eve.config.peers = node::config::PeerConfig::Static;
     let mut eve = eve.spawn();
     alice.handle.seed(acme, Scope::All).unwrap();
     bob.handle.seed(acme, Scope::All).unwrap();
     bob.connect(&alice).converge([&alice]);
     eve.connect(&alice);
     eve.connect(&bob);
     eve.routes_to(&[(acme, carol), (acme, bob.id), (acme, alice.id)]);
     bob.handle.unseed(acme).unwrap(); // Cause the fetch with bob to fail.
     test(
         "examples/rad-clone-partial-fail.md",
         working.join("eve"),
         Some(&eve.home),
         [],
     )
     .unwrap();
 }
 #[test]
 fn rad_clone_connect() {
     let mut environment = Environment::new();
     let working = environment.tmp().join("working");

@@ -2017,6 +2074,44 @@ fn rad_init_private() {

 }
 #[test]
 fn rad_init_private_seed() {
     let mut environment = Environment::new();
     let alice = environment.node(Config::test(Alias::new("alice")));
     let bob = environment.node(Config::test(Alias::new("bob")));
     let working = environment.tmp().join("working");
     fixtures::repository(working.join("alice"));
     let alice = alice.spawn();
     let mut bob = bob.spawn();
     test(
         "examples/rad-init-private.md",
         working.join("alice"),
         Some(&alice.home),
         [],
     )
     .unwrap();
     bob.connect(&alice).converge([&alice]);
     formula(&environment.tmp(), "examples/rad-init-private-seed.md")
         .unwrap()
         .home(
             "alice",
             working.join("alice"),
             [("RAD_HOME", alice.home.path().display())],
         )
         .home(
             "bob",
             bob.home.path(),
             [("RAD_HOME", bob.home.path().display())],
         )
         .run()
         .unwrap();
 }
 #[test]
 fn rad_init_private_clone() {
     let mut environment = Environment::new();
     let alice = environment.node(Config::test(Alias::new("alice")));