Radish alpha
r
rad:z3qg5TKmN83afz2fj9z3fQjU8vaYE
Radicle CI adapter for native CI
Radicle
Git
feat: produce an index.html with list of, and links to, run logs
Lars Wirzenius committed 2 years ago
commit d5b88b6596b8a876be7655bd97b0dfce599e4688
parent 28a9310
4 files changed +382 -11
modified Cargo.lock
@@ -343,6 +343,15 @@ dependencies = [
]

[[package]]
+
name = "deranged"
+
version = "0.3.10"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
+
dependencies = [
+
 "powerfmt",
+
]
+

+
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -592,6 +601,25 @@ dependencies = [
]

[[package]]
+
name = "html-escape"
+
version = "0.2.13"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
+
dependencies = [
+
 "utf8-width",
+
]
+

+
[[package]]
+
name = "html-page"
+
version = "0.1.0"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "3f8314b0ea57e9e3fc648213a02315e8a16154bb86da7516fec7a09ec4d7417c"
+
dependencies = [
+
 "html-escape",
+
 "line-col",
+
]
+

+
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -707,6 +735,12 @@ dependencies = [
]

[[package]]
+
name = "line-col"
+
version = "0.2.1"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "9e69cdf6b85b5c8dce514f694089a2cf8b1a702f6cd28607bcb3cf296c9778db"
+

+
[[package]]
name = "linux-raw-sys"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -888,6 +922,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"

[[package]]
+
name = "powerfmt"
+
version = "0.2.0"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+

+
[[package]]
name = "ppv-lite86"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1049,6 +1089,7 @@ dependencies = [
name = "radicle-native-ci"
version = "0.1.0"
dependencies = [
+
 "html-page",
 "log",
 "pretty_env_logger",
 "radicle",
@@ -1059,7 +1100,9 @@ dependencies = [
 "subprocess",
 "tempfile",
 "thiserror",
+
 "time",
 "uuid",
+
 "walkdir",
]

[[package]]
@@ -1239,6 +1282,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"

[[package]]
+
name = "same-file"
+
version = "1.0.6"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+
dependencies = [
+
 "winapi-util",
+
]
+

+
[[package]]
name = "sec1"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1522,6 +1574,35 @@ dependencies = [
]

[[package]]
+
name = "time"
+
version = "0.3.31"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e"
+
dependencies = [
+
 "deranged",
+
 "itoa",
+
 "powerfmt",
+
 "serde",
+
 "time-core",
+
 "time-macros",
+
]
+

+
[[package]]
+
name = "time-core"
+
version = "0.1.2"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+

+
[[package]]
+
name = "time-macros"
+
version = "0.2.16"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f"
+
dependencies = [
+
 "time-core",
+
]
+

+
[[package]]
name = "tinyvec"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1581,6 +1662,12 @@ dependencies = [
]

[[package]]
+
name = "utf8-width"
+
version = "0.1.7"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
+

+
[[package]]
name = "uuid"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1602,6 +1689,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"

[[package]]
+
name = "walkdir"
+
version = "2.4.0"
+
source = "registry+https://github.com/rust-lang/crates.io-index"
+
checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
+
dependencies = [
+
 "same-file",
+
 "winapi-util",
+
]
+

+
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
modified Cargo.toml
@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2021"

[dependencies]
+
html-page = "0.1.0"
log = "0.4.20"
pretty_env_logger = "0.5.0"
radicle-git-ext = "0.7.0"
@@ -12,7 +13,9 @@ serde_yaml = "0.9.27"
subprocess = "0.2.9"
tempfile = "3.8.1"
thiserror = "1.0.50"
+
time = { version = "0.3.31", features = ["formatting", "macros"] }
uuid = { version = "1.6.1", features = ["v4"] }
+
walkdir = "2.4.0"

[dependencies.radicle]
git = "https://seed.radicle.xyz/z3gqcJUoA1n9HaHKufZs5FCSGazv5.git"
modified src/main.rs
@@ -21,13 +21,16 @@ use std::{
};

use log::{debug, info};
-
use serde::Deserialize;
+
use serde::{Deserialize, Serialize};
use subprocess::{Popen, PopenConfig, Redirection};
+
use time::{macros::format_description, OffsetDateTime};
use uuid::Uuid;

use radicle::prelude::Profile;
use radicle_ci_broker::msg::{Id, MessageError, Oid, Request, Response, RunId, RunResult};

+
mod report;
+

/// Path to the repository's CI run specification. This is relative to
/// the root of the repository.
const RUNSPEC_PATH: &str = ".radicle/native.yaml";
@@ -54,22 +57,42 @@ fn fallible_main() -> Result<(), NativeError> {
    logfile.write("radicle-native-ci starts\n".into())?;

    let (run_id, run_dir) = mkdir_run(&config)?;
+
    let run_id = RunId::from(format!("{}", run_id).as_str());
    logfile.write(format!("run directory {}\n", run_dir.display()))?;

    let src = run_dir.join("src");
    let run_log = run_dir.join("log.txt");
+
    let run_info_file = run_dir.join("run.yaml");

    let profile = Profile::load().map_err(NativeError::LoadProfile)?;
    let storage = profile.storage.path();

    let req = read_request()?;

-
    if let Request::Trigger { repo, commit } = req {
+
    let run_info = if let Request::Trigger { repo, commit } = req {
        info!("Request to trigger CI on {}, {}", repo, commit);
-
        run(run_id, storage, repo, commit, &src, &mut logfile, &run_log)?;
+
        run(
+
            run_id,
+
            storage,
+
            repo,
+
            commit,
+
            &src,
+
            &mut logfile,
+
            &run_log,
+
            &config.state,
+
        )?
    } else {
        write_response(&Response::error("first request was not Trigger\n"))?;
-
    }
+
        RunInfo::builder()
+
            .id(run_id)
+
            .log(&config.state, run_log)
+
            .result(RunResult::Error("first request was not Trigger".into()))
+
            .build()?
+
    };
+

+
    run_info.write(&run_info_file)?;
+

+
    report::build_report(&config.state)?;

    logfile.write("radicle-native-ci ends successfully".into())?;
    info!("radicle-native-ci ends");
@@ -106,15 +129,17 @@ fn write_response(resp: &Response) -> Result<(), NativeError> {
}

/// Perform the CI run.
+
#[allow(clippy::too_many_arguments)]
fn run(
-
    run_id: Uuid,
+
    run_id: RunId,
    storage: &Path,
    repo: Id,
    commit: Oid,
    src: &Path,
    log: &mut LogFile,
    run_log: &Path,
-
) -> Result<(), NativeError> {
+
    state: &Path,
+
) -> Result<RunInfo, NativeError> {
    let mut run_log = LogFile::open(run_log)?;

    log.write(format!("CI run on {}, {}\n", repo, commit))?;
@@ -127,8 +152,8 @@ fn run(
        run_id.to_string().as_str(),
    )))?;

-
    let repo = storage.join(repo.canonical());
-
    debug!("repo path: {}", repo.display());
+
    let repo_path = storage.join(repo.canonical());
+
    debug!("repo path: {}", repo_path.display());

    debug!("cloning repository to {}", src.display());
    runcmd(
@@ -136,7 +161,7 @@ fn run(
        &[
            "git",
            "clone",
-
            repo.to_str().unwrap(),
+
            repo_path.to_str().unwrap(),
            src.to_str().unwrap(),
        ],
        storage,
@@ -152,13 +177,23 @@ fn run(
    let snippet = format!("set -xeuo pipefail\n{}", &runspec.shell);
    runcmd(&mut run_log, &["bash", "-c", &snippet], src)?;

-
    write_response(&Response::finished(RunResult::Success))?;
+
    let result = RunResult::Success;
+

+
    write_response(&Response::finished(result.clone()))?;

    run_log.write("CI run finished successfully\n".into())?;

    std::fs::remove_dir_all(src).map_err(|e| NativeError::RemoveDir(src.into(), e))?;

-
    Ok(())
+
    let info = RunInfo::builder()
+
        .repo(repo)
+
        .commit(commit)
+
        .id(run_id)
+
        .result(result)
+
        .log(state, run_log.filename().into())
+
        .build()?;
+

+
    Ok(info)
}

/// Run a command in a directory.
@@ -239,6 +274,10 @@ struct LogFile {
}

impl LogFile {
+
    fn filename(&self) -> &Path {
+
        self.filename.as_path()
+
    }
+

    fn open(filename: &Path) -> Result<Self, NativeError> {
        let file = std::fs::OpenOptions::new()
            .append(true)
@@ -291,6 +330,99 @@ impl RunSpec {
    }
}

+
/// Metadata about a run.
+
#[derive(Debug, Serialize, Deserialize)]
+
#[serde(deny_unknown_fields)]
+
#[allow(dead_code)]
+
pub struct RunInfo {
+
    /// Repository ID.
+
    repo: String,
+

+
    /// Commit ID.
+
    commit: String,
+

+
    /// Identifier for the CI run.
+
    id: String,
+

+
    /// Result of the CI run.
+
    result: RunResult,
+

+
    /// Name of log file.
+
    log: PathBuf,
+

+
    /// Timestamp of when the run ended (the value was created).
+
    /// ISO8601 format.
+
    timestamp: String,
+
}
+

+
impl RunInfo {
+
    fn builder() -> RunInfoBuilder {
+
        RunInfoBuilder::default()
+
    }
+

+
    fn write(&self, filename: &Path) -> Result<(), NativeError> {
+
        let yaml = serde_yaml::to_string(&self).map_err(NativeError::SerializeRunInfo)?;
+
        std::fs::write(filename, yaml.as_bytes())
+
            .map_err(|e| NativeError::WriteRunInfo(filename.into(), e))?;
+
        Ok(())
+
    }
+
}
+

+
#[derive(Debug, Default)]
+
#[allow(dead_code)]
+
struct RunInfoBuilder {
+
    repo: Option<Id>,
+
    commit: Option<Oid>,
+
    id: Option<String>,
+
    result: Option<RunResult>,
+
    log: Option<PathBuf>,
+
}
+

+
impl RunInfoBuilder {
+
    fn repo(mut self, repo: Id) -> Self {
+
        self.repo = Some(repo);
+
        self
+
    }
+

+
    fn commit(mut self, commit: Oid) -> Self {
+
        self.commit = Some(commit);
+
        self
+
    }
+

+
    fn id(mut self, id: RunId) -> Self {
+
        self.id = Some(format!("{}", id));
+
        self
+
    }
+

+
    fn result(mut self, result: RunResult) -> Self {
+
        self.result = Some(result);
+
        self
+
    }
+

+
    fn log(mut self, state: &Path, log: PathBuf) -> Self {
+
        self.log = Some(log.strip_prefix(state).unwrap().into());
+
        self
+
    }
+

+
    fn build(self) -> Result<RunInfo, NativeError> {
+
        let fmt = format_description!("[year]-[month]-[day] [hour]:[minute]:[second]");
+
        let now = OffsetDateTime::now_utc()
+
            .format(fmt)
+
            .map_err(NativeError::TimeFormat)?;
+

+
        Ok(RunInfo {
+
            repo: self.repo.map(|x| x.to_string()).unwrap_or("".into()),
+
            commit: self.commit.map(|x| x.to_string()).unwrap_or("".into()),
+
            id: self.id.ok_or(NativeError::MissingInfo("id".into()))?,
+
            result: self
+
                .result
+
                .ok_or(NativeError::MissingInfo("result".into()))?,
+
            log: self.log.ok_or(NativeError::MissingInfo("log".into()))?,
+
            timestamp: now,
+
        })
+
    }
+
}
+

#[derive(Debug, thiserror::Error)]
enum NativeError {
    #[error("failed to read configuration file {0}")]
@@ -340,4 +472,19 @@ enum NativeError {

    #[error("failed to remove {0}")]
    RemoveDir(PathBuf, #[source] std::io::Error),
+

+
    #[error("programming error: field {0} is not set")]
+
    MissingInfo(String),
+

+
    #[error("failed to write file for run metadata: {0}")]
+
    WriteRunInfo(PathBuf, #[source] std::io::Error),
+

+
    #[error("failed to serialize run metadata: programming error")]
+
    SerializeRunInfo(#[source] serde_yaml::Error),
+

+
    #[error(transparent)]
+
    Report(#[from] report::ReportError),
+

+
    #[error("failed to format current time as a string")]
+
    TimeFormat(#[source] time::error::Format),
}
added src/report.rs
@@ -0,0 +1,124 @@
+
use std::{
+
    collections::HashSet,
+
    path::{Path, PathBuf},
+
};
+

+
use html_page::{Document, Element, Tag};
+
use radicle_ci_broker::msg::RunResult;
+
use walkdir::WalkDir;
+

+
use crate::RunInfo;
+

+
const CSS: &str = include_str!("native-ci.css");
+

+
pub fn build_report(state: &Path) -> Result<(), ReportError> {
+
    let mut run_infos = collect_run_infos(state)?;
+
    run_infos.sort_by(|a, b| a.timestamp.partial_cmp(&b.timestamp).unwrap());
+

+
    let repos = repositories(&run_infos);
+
    let doc = list_runs(&repos, &run_infos);
+

+
    let index = state.join("index.html");
+
    std::fs::write(&index, doc.to_string().as_bytes())
+
        .map_err(|e| ReportError::WriteHtml(index.clone(), e))?;
+

+
    Ok(())
+
}
+

+
fn collect_run_infos(state: &Path) -> Result<Vec<RunInfo>, ReportError> {
+
    let mut infos = vec![];
+
    for entry in WalkDir::new(state) {
+
        let entry = entry?;
+
        if entry.path().ends_with("run.yaml") {
+
            infos.push(read_file_info(entry.path())?);
+
        }
+
    }
+
    Ok(infos)
+
}
+

+
fn repositories(infos: &[RunInfo]) -> Vec<&str> {
+
    let mut ids: Vec<&str> = infos
+
        .iter()
+
        .map(|ri| ri.repo.as_str())
+
        .collect::<HashSet<&str>>()
+
        .iter()
+
        .copied()
+
        .collect();
+
    ids.sort_by(|a, b| a.partial_cmp(b).unwrap());
+
    ids
+
}
+

+
fn runs_for_repo<'a>(repo: &str, infos: &'a [RunInfo]) -> Vec<&'a RunInfo> {
+
    infos.iter().filter(|ri| ri.repo == repo).collect()
+
}
+

+
fn read_file_info(filename: &Path) -> Result<RunInfo, ReportError> {
+
    let yaml = std::fs::read(filename).map_err(|e| ReportError::ReadRunInfo(filename.into(), e))?;
+
    serde_yaml::from_slice(&yaml).map_err(|e| ReportError::DeserializeRunInfo(filename.into(), e))
+
}
+

+
fn list_runs(repos: &[&str], run_infos: &[RunInfo]) -> Document {
+
    let mut doc = Document::default();
+
    doc.push_to_head(&Element::new(Tag::Title).with_text("CI run logs"));
+
    doc.push_to_head(&Element::new(Tag::Style).with_text(CSS));
+

+
    for repo in repos {
+
        let section = Element::new(Tag::H1)
+
            .with_text("Repository ")
+
            .with_child(Element::new(Tag::Code).with_text(repo));
+
        doc.push_to_body(&section);
+

+
        let mut list = Element::new(Tag::Ol).with_class("runlist");
+
        for ri in runs_for_repo(repo, run_infos) {
+
            let commit = Element::new(Tag::Code)
+
                .with_class("commit")
+
                .with_text(&ri.commit);
+

+
            let timestamp = Element::new(Tag::Span)
+
                .with_class("timestamp")
+
                .with_text(&ri.timestamp);
+

+
            let (result_text, class) = match &ri.result {
+
                RunResult::Success => ("success".into(), "success"),
+
                RunResult::Failure => ("failure".into(), "failure"),
+
                RunResult::Error(msg) => (format!("error: {}", msg), "failure"),
+
                _ => (format!("unknown: {:?}", ri.result), "unknown"),
+
            };
+
            let result = Element::new(Tag::Span)
+
                .with_class(class)
+
                .with_text(&result_text);
+

+
            let br = Element::new(Tag::Br);
+

+
            let href = format!("{}", ri.log.display());
+
            let link = Element::new(Tag::A)
+
                .with_attribute("href", &href)
+
                .with_child(timestamp)
+
                .with_text(" ")
+
                .with_child(result)
+
                .with_child(br)
+
                .with_child(commit);
+

+
            let item = Element::new(Tag::Li).with_child(link);
+
            list.push_child(&item);
+
        }
+
        doc.push_to_body(&list);
+
    }
+

+
    doc
+
}
+

+
#[derive(Debug, thiserror::Error)]
+
pub enum ReportError {
+
    #[error(transparent)]
+
    WalkDir(#[from] walkdir::Error),
+

+
    #[error("failed to read file for run metadata: {0}")]
+
    ReadRunInfo(PathBuf, #[source] std::io::Error),
+

+
    #[error("failed to parse run metadata as YAML: {0}")]
+
    DeserializeRunInfo(PathBuf, #[source] serde_yaml::Error),
+

+
    #[error("failed to write HTML file {0}")]
+
    WriteHtml(PathBuf, #[source] std::io::Error),
+
}