Radish alpha
h
rad:z3gqcJUoA1n9HaHKufZs5FCSGazv5
Radicle Heartwood Protocol & Stack
Radicle
Git
node: Update sync status for private repos
Merged did:key:z6MksFqX...wzpT opened 1 year ago

Some improvements to the output of rad sync status for private repos.

10 files changed +82 -44 9f36320d 191278f0
modified radicle-cli/examples/rad-sync.md
@@ -7,14 +7,15 @@ For instance let's create an issue and sync it with the network:
$ rad issue open --title "Test `rad sync`" --description "Check that the command works" -q --no-announce
```

-
If we check the sync status, we see that our peers are out of sync.
+
If we check the sync status, we see that our peers are out of sync, and our
+
change has not yet been announced.

```
$ rad sync status --sort-by alias
╭──────────────────────────────────────────────────────────────────────────────────────────────╮
│ ●   Node                      Address                      Status        Tip       Timestamp │
├──────────────────────────────────────────────────────────────────────────────────────────────┤
-
│ ●   alice   (you)             alice.radicle.example:8776                 f209c9f   [  ...  ] │
+
│ ●   alice   (you)             alice.radicle.example:8776   unannounced   a9ce0d1   [  ...  ] │
│ ●   bob     z6Mkt67…v4N1tRk   bob.radicle.example:8776     out-of-sync   f209c9f   [  ...  ] │
│ ●   eve     z6Mkux1…nVhib7Z   eve.radicle.example:8776     out-of-sync   f209c9f   [  ...  ] │
╰──────────────────────────────────────────────────────────────────────────────────────────────╯
modified radicle-cli/src/commands/ls.rs
@@ -102,6 +102,7 @@ pub fn run(options: Options, ctx: impl term::Context) -> anyhow::Result<()> {
        head,
        doc,
        refs,
+
        ..
    } in repos
    {
        if doc.visibility.is_public() && options.private && !options.public {
modified radicle-cli/src/commands/sync.rs
@@ -317,7 +317,7 @@ fn sync_status(
) -> anyhow::Result<()> {
    let mut table = Table::<7, term::Label>::new(TableOptions::bordered());
    let mut seeds: Vec<_> = node.seeds(rid)?.into();
-
    let local = node.nid()?;
+
    let local_nid = node.nid()?;
    let aliases = profile.aliases();

    table.push([
@@ -331,20 +331,32 @@ fn sync_status(
    ]);
    table.divider();

-
    sort_seeds_by(local, &mut seeds, &aliases, &options.sort_by);
+
    sort_seeds_by(local_nid, &mut seeds, &aliases, &options.sort_by);

    for seed in seeds {
        let (icon, status, head, time) = match seed.sync {
            Some(SyncStatus::Synced { at }) => (
                term::format::positive("●"),
-
                term::format::positive(if seed.nid != local { "synced" } else { "" }),
+
                term::format::positive(if seed.nid != local_nid { "synced" } else { "" }),
                term::format::oid(at.oid),
                term::format::timestamp(at.timestamp),
            ),
-
            Some(SyncStatus::OutOfSync { remote, .. }) => (
-
                term::format::negative("●"),
-
                term::format::negative(if seed.nid != local { "out-of-sync" } else { "" }),
-
                term::format::oid(remote.oid),
+
            Some(SyncStatus::OutOfSync { remote, local, .. }) => (
+
                if seed.nid != local_nid {
+
                    term::format::negative("●")
+
                } else {
+
                    term::format::yellow("●")
+
                },
+
                if seed.nid != local_nid {
+
                    term::format::negative("out-of-sync")
+
                } else {
+
                    term::format::yellow("unannounced")
+
                },
+
                term::format::oid(if seed.nid != local_nid {
+
                    remote.oid
+
                } else {
+
                    local.oid
+
                }),
                term::format::timestamp(remote.timestamp),
            ),
            None if options.verbose => (
modified radicle-node/src/service.rs
@@ -587,11 +587,9 @@ where
        assert_ne!(time, LocalTime::default());

        let nid = self.node_id();
-
        let inventory = self.storage.inventory()?;

        self.started_at = Some(time);
        self.clock = time;
-
        self.inventory = gossip::inventory(self.timestamp(), inventory.clone());

        // Populate refs database. This is only useful as part of the upgrade process for nodes
        // that have been online since before the refs database was created.
@@ -622,31 +620,27 @@ where
            )
            .expect("Service::initialize: error adding local node to address database");

-
        // Ensure that our inventory is recorded in our routing table, and we are seeding
-
        // all of it. It can happen that inventory is not properly seeded if for eg. the
-
        // user creates a new repository while the node is stopped.
-
        self.db
-
            .routing_mut()
-
            .insert(inventory.iter(), nid, time.into())?;
-

        let announced = self
            .db
            .seeds()
            .seeded_by(&nid)?
            .collect::<Result<HashMap<_, _>, _>>()?;
-
        for rid in inventory {
-
            let repo = self.storage.repository(rid)?;
+
        for repo in self.storage.repositories()? {
+
            let rid = repo.rid;

            // If we're not seeding this repo, just skip it.
            if !self.policies.is_seeding(&rid)? {
                warn!(target: "service", "Local repository {rid} is not seeded");
                continue;
            }
+
            // Add public repositories to inventory.
+
            if repo.doc.visibility.is_public() {
+
                self.storage.insert(rid);
+
            }
            // If we have no owned refs for this repo, then there's nothing to announce.
-
            let Ok(updated_at) = SyncedAt::load(&repo, nid) else {
+
            let Some(updated_at) = repo.synced_at else {
                continue;
            };
-

            // Skip this repo if the sync status matches what we have in storage.
            if let Some(announced) = announced.get(&rid) {
                if updated_at.oid == announced.oid {
@@ -662,7 +656,6 @@ where
            )? {
                debug!(target: "service", "Saved local sync status for {rid}..");
            }
-

            // If we got here, it likely means a repo was updated while the node was stopped.
            // Therefore, we pre-load a refs announcement for this repo, so that it is included in
            // the historical gossip messages when a node connects and subscribes to this repo.
@@ -672,6 +665,17 @@ where
            }
        }

+
        {
+
            let inventory = self.storage.inventory()?;
+
            // Ensure that our inventory is recorded in our routing table, and we are seeding
+
            // all of it. It can happen that inventory is not properly seeded if for eg. the
+
            // user creates a new repository while the node is stopped.
+
            self.db
+
                .routing_mut()
+
                .insert(inventory.iter(), nid, time.into())?;
+
            self.inventory = gossip::inventory(self.timestamp(), inventory);
+
        }
+

        // Setup subscription filter for seeded repos.
        self.filter = Filter::new(
            self.policies
@@ -1983,20 +1987,21 @@ where

        // Update our sync status for our own refs. This is useful for determining if refs were
        // updated while the node was stopped.
-
        // TODO: Move to `announce_own_refs`.
        if let Some(refs) = refs.iter().find(|r| r.remote == ann.node) {
            info!(
                target: "service",
                "Announcing own refs for {rid} to peers ({}) (t={timestamp})..",
                refs.at
            );
-

+
            // Update our local node's sync status to mark the refs as announced.
            if let Err(e) = self
                .db
                .seeds_mut()
                .synced(&rid, &ann.node, refs.at, timestamp)
            {
                error!(target: "service", "Error updating sync status for local node: {e}");
+
            } else {
+
                debug!(target: "service", "Saved local sync status for {rid}..");
            }
        }

modified radicle-node/src/tests.rs
@@ -838,7 +838,6 @@ fn test_refs_announcement_no_subscribe() {

#[test]
fn test_refs_announcement_offline() {
-
    logger::init(log::Level::Debug);
    let tmp = tempfile::tempdir().unwrap();
    let mut alice = {
        let signer = MockSigner::default();
@@ -854,10 +853,10 @@ fn test_refs_announcement_offline() {
            },
        )
    };
-
    let inv = alice.inventory();
-
    let rid = inv.first().unwrap();
+
    let mut inv = alice.inventory();
+
    let rid = *inv.first().unwrap();
    let mut bob = Peer::new("bob", [8, 8, 8, 8]);
-
    bob.seed(rid, policy::Scope::All).unwrap();
+
    bob.seed(&rid, policy::Scope::All).unwrap();

    // Make sure alice's service wasn't initialized before.
    assert_eq!(*alice.clock(), LocalTime::default());
@@ -868,25 +867,23 @@ fn test_refs_announcement_offline() {

    // Alice announces the refs of all projects since she hasn't announced refs for these projects
    // yet.
-
    let mut messages = alice.messages(bob.id());
-
    for i in &inv {
-
        let msg = messages.next();
+
    for msg in alice.messages(bob.id()) {
        assert_matches!(
            msg,
-
            Some(Message::Announcement(Announcement {
+
            Message::Announcement(Announcement {
                node,
                message: AnnouncementMessage::Refs(RefsAnnouncement {
                    rid,
                    ..
                }),
                ..
-
            }))
-
            if node == alice.id && rid == *i
+
            })
+
            if node == alice.id && inv.remove(&rid)
        );
    }

    // Create an issue without telling the node.
-
    let repo = alice.storage().repository(*rid).unwrap();
+
    let repo = alice.storage().repository(rid).unwrap();
    let old_refs = RefsAt::new(&repo, alice.id).unwrap();
    let mut issues = radicle::issue::Cache::no_cache(&repo).unwrap();
    issues
@@ -935,7 +932,7 @@ fn test_refs_announcement_offline() {
        .collect::<Vec<_>>();

    assert_eq!(anns.len(), 1);
-
    assert_eq!(anns.first().unwrap().rid, *rid);
+
    assert_eq!(anns.first().unwrap().rid, rid);
    assert_eq!(anns.first().unwrap().refs.first().unwrap().at, new_refs.at);
}

modified radicle/src/storage.rs
@@ -21,6 +21,7 @@ use crate::git::{refspec::Refspec, PatternString, Qualified, RefError, RefStr, R
use crate::identity::{Did, PayloadError};
use crate::identity::{Doc, DocAt, DocError};
use crate::identity::{Identity, RepoId};
+
use crate::node::SyncedAt;
use crate::storage::git::NAMESPACES_GLOB;
use crate::storage::refs::Refs;

@@ -42,6 +43,8 @@ pub struct RepositoryInfo<V> {
    /// Local signed refs, if any.
    /// Repositories with this set to `None` are ones that are seeded but not forked.
    pub refs: Option<refs::SignedRefsAt>,
+
    /// Sync time of the repository.
+
    pub synced_at: Option<SyncedAt>,
}

/// Describes one or more namespaces.
modified radicle/src/storage/git.rs
@@ -13,16 +13,17 @@ use once_cell::sync::Lazy;
use tempfile::TempDir;

use crate::crypto::Unverified;
-
use crate::git;
use crate::identity::doc::DocError;
use crate::identity::{doc::DocAt, Doc, RepoId};
use crate::identity::{Identity, Project};
+
use crate::node::SyncedAt;
use crate::storage::refs;
use crate::storage::refs::{Refs, SignedRefs, SignedRefsAt};
use crate::storage::{
    Inventory, ReadRepository, ReadStorage, Remote, Remotes, RepositoryError, RepositoryInfo,
    SetHead, SignRepository, WriteRepository, WriteStorage,
};
+
use crate::{git, node};

pub use crate::git::{
    ext, raw, refname, refspec, Oid, PatternStr, Qualified, RefError, RefString, UserInfo,
@@ -152,10 +153,10 @@ impl ReadStorage for Storage {
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());

-
        // If the cache hasn't been populated yet, we don't do anything, since this repo
-
        // will be loaded when the cache is populated.
        if let Some(ref mut repos) = *repos {
            repos.insert(rid);
+
        } else {
+
            *repos = Some(BTreeSet::from_iter([rid]));
        }
    }

@@ -211,12 +212,17 @@ impl ReadStorage for Storage {
            };
            // Nb. This will be `None` if they were not found.
            let refs = refs::SignedRefsAt::load(self.info.key, &repo)?;
+
            let synced_at = refs
+
                .as_ref()
+
                .map(|r| node::SyncedAt::new(r.at, &repo))
+
                .transpose()?;

            repos.push(RepositoryInfo {
                rid,
                head,
                doc,
                refs,
+
                synced_at,
            });
        }
        Ok(repos)
@@ -298,11 +304,17 @@ impl Storage {
        rids.try_fold(Vec::new(), |mut infos, rid| {
            let repo = self.repository(*rid)?;
            let (_, head) = repo.head()?;
+
            let refs = refs::SignedRefsAt::load(self.info.key, &repo)?;
+
            let synced_at = refs
+
                .as_ref()
+
                .map(|r| SyncedAt::new(r.at, &repo))
+
                .transpose()?;
            let info = RepositoryInfo {
                rid: *rid,
                head,
                doc: repo.identity_doc()?.into(),
-
                refs: refs::SignedRefsAt::load(self.info.key, &repo)?,
+
                refs,
+
                synced_at,
            };
            infos.push(info);
            Ok(infos)
modified radicle/src/storage/refs.rs
@@ -411,8 +411,8 @@ impl SignedRefsAt {
    where
        S: ReadRepository,
    {
-
        let at = match repo.reference_oid(&remote, &SIGREFS_BRANCH) {
-
            Ok(at) => at,
+
        let at = match RefsAt::new(repo, remote) {
+
            Ok(RefsAt { at, .. }) => at,
            Err(e) if git::is_not_found_err(&e) => return Ok(None),
            Err(e) => return Err(e.into()),
        };
modified radicle/src/test/fixtures.rs
@@ -25,7 +25,13 @@ pub fn user() -> git::UserInfo {
/// Create a new storage with a project.
pub fn storage<P: AsRef<Path>, G: Signer>(path: P, signer: &G) -> Result<Storage, rad::InitError> {
    let path = path.as_ref();
-
    let storage = Storage::open(path.join("storage"), user())?;
+
    let storage = Storage::open(
+
        path.join("storage"),
+
        git::UserInfo {
+
            alias: Alias::new("Radcliff"),
+
            key: *signer.public_key(),
+
        },
+
    )?;

    transport::local::register(storage.clone());
    transport::remote::mock::register(signer.public_key(), storage.path());
modified radicle/src/test/storage.rs
@@ -118,6 +118,7 @@ impl ReadStorage for MockStorage {
                head: r.head().unwrap().1,
                doc: r.doc.clone().into(),
                refs: None,
+
                synced_at: None,
            })
            .collect())
    }