Radish alpha
r
radicle-ci-broker
Radicle CI broker
Radicle
Git (anonymous pull)
Log in to clone via SSH
Commits ›
tests: add subplot to capture acceptance criteria and verification
Lars Wirzenius committed 2 years ago
commit 1f1e0c72b7a946ae09837571e2d7cbf8349a4ec8
parent c3eda6c3f1ec37fda3fba6857985813f1dfecc00
6 files changed +406 -0
modified build.rs
@@ -1,3 +1,5 @@
+ 
// Build acceptance test suite using Subplot.
+ 


use std::process::Command;



fn main() {
@@ -20,4 +22,6 @@ fn main() {


    println!("cargo:rustc-env=GIT_HEAD={hash}");

    println!("cargo:rustc-rerun-if-changed=.git/HEAD");
+
+ 
    subplot_build::codegen("ci-broker.subplot").expect("failed to generate code with Subplot");

}
added ci-broker.md
@@ -0,0 +1,330 @@
+ 
# Introduction
+
+ 
This document describes the acceptance criteria for the Radicle CI
+ 
broker, as well as how to verify that they are met. Acceptance
+ 
criteria here means a requirement that must be met for the software to
+ 
be acceptable to its stakeholders.
+
+ 
This file is used by [Subplot](https://subplot.tech/) to generate and
+ 
run test code as part of running `cargo test`.
+
+ 
# Data files shared between scenarios
+
+ 
## Broker configuration
+
+ 
~~~{#broker.yaml .file .yaml}
+ 
db: ci-broker.db
+ 
report_dir: reports
+ 
default_adapter: dummy
+ 
adapters:
+ 
  dummy:
+ 
    command: ./dummy.sh
+ 
    env:
+ 
      RADICLE_NATIVE_CI: native-ci.yaml
+ 
    sensitive_env: {}
+ 
filters:
+ 
  - !Branch "main"
+ 
~~~
+
+ 
## A dummy adapter
+
+ 
This adapter does nothing, just reports a run ID and a successful run.
+
+ 
~~~{#dummy.sh .file .sh}
+ 
#!/bin/bash
+ 
set -euo pipefail
+ 
echo '{"response":"triggered","run_id":{"id":"xyzzy"}}'
+ 
echo '{"response":"finished","result":"success"}'
+ 
~~~
+
+ 
## A trigger message
+
+ 
This is a request message from the CI broker to the adapter to trigger
+ 
a run on a repository. The repository is imaginary as is the commit.
+
+ 
~~~{#trigger.json .file .json}
+ 
{
+ 
  "type": "refsFetched",
+ 
  "remote": "z6MkgEMYod7Hxfy9qCvDv5hYHkZ4ciWmLFgfvm3Wn1b2w2FV",
+ 
  "rid": "rad:zwTxygwuz5LDGBq255RA2CbNGrz8",
+ 
  "updated": [
+ 
    {
+ 
      "updated": {
+ 
        "name": "refs/heads/main",
+ 
        "old": "0000000000000000000000000000000000000000",
+ 
        "new": "0000000000000000000000000000000000000000"
+ 
      }
+ 
    }
+ 
  ]
+ 
}
+ 
~~~
+
+ 
## Set rid in trigger message
+
+ 
This is a helper script that reads a trigger message and changes its
+ 
`rid` field to be the id of the given repository. It also sets the
+ 
`name` field for updated refs to include the repository ID. It writes
+ 
the message back to its file.
+
+ 
~~~{#set-rid .file .python}
+ 
#!/usr/bin/python3
+
+ 
import json, sys
+ 
from subprocess import run, PIPE
+
+ 
filename = sys.argv[1]
+ 
cwd = sys.argv[2]
+
+ 
p = run(["rad", "."], check=True, capture_output=True, cwd=cwd)
+ 
rid = p.stdout.decode().strip()
+
+ 
p = run(["rad", "self", "--nid"], check=True, capture_output=True, cwd=cwd)
+ 
nid = p.stdout.decode().strip()
+
+ 
p = run(["git", "rev-parse", "HEAD"], check=True, capture_output=True, cwd=cwd)
+ 
oid = p.stdout.decode().strip()
+
+ 
o = json.load(open(filename))
+
+ 
o["rid"] = rid
+
+ 
if "updated" in o:
+ 
    x = o["updated"]
+ 
    for oo in x:
+ 
        name = oo["updated"]["name"]
+ 
        oo["updated"]["name"] = f"refs/namespaces/{nid}/{name}"
+ 
        oo["updated"]["new"] = oid
+ 
    o["updated"] = x
+
+ 
with open(filename, "w") as f:
+ 
    json.dump(o, fp=f, indent=4)
+ 
~~~
+
+ 
# Acceptance criteria
+
+ 
## Smoke test: Runs adapter
+
+ 
_Requirement:_ CI broker can run its adapter.
+
+ 
_Justification:_ This is obviously necessary.
+
+ 
_Stakeholder:_ Lars.
+
+ 
~~~scenario
+ 
given a directory homedir
+ 
when I run env HOME=homedir RAD_PASSPHRASE= rad auth --alias brokertest
+
+ 
when I run env HOME=homedir git config --global user.email radicle@example.com
+ 
when I run env HOME=homedir git config --global user.name TestyMcTestFace
+ 
when I run env HOME=homedir git init testy
+ 
given file testy/test.txt from dummy.sh
+ 
when I run, in testy, env HOME=../homedir git add .
+ 
when I run, in testy, env HOME=../homedir git commit -am test
+ 
when I run, in testy, env HOME=../homedir git status
+ 
when I run, in testy, env HOME=../homedir RAD_PASSPHRASE= rad init --name testy --description test --default-branch master --private --no-confirm --no-seed
+ 
when I run, in testy, env HOME=../homedir rad .
+
+ 
given an installed synthetic-events
+ 
given file trigger.json
+ 
given file shutdown.json
+ 
given file set-rid
+ 
when I run env HOME=../homedir python3 set-rid trigger.json testy
+ 
when I run synthetic-events synt.sock trigger.json shutdown.json
+
+ 
given an installed ci-broker
+ 
given file broker.yaml
+ 
given file dummy.sh
+ 
when I run chmod +x dummy.sh
+
+ 
when I run env HOME=homedir rad ls --all
+ 
when I run env HOME=homedir rad self
+ 
when I run find homedir -type f
+ 
when I run cat trigger.json
+ 
when I run sed -i 's/"auto"/false/' homedir/.radicle/config.json
+ 
when I run env HOME=homedir RAD_SOCKET=synt.sock RUST_LOG=debug ci-broker broker.yaml
+ 
then command is successful
+ 
~~~
+
+
+ 
## Gives helpful error message if node socket can't be found
+
+ 
_Requirement:_ If the CI broker can't connect to the Radicle node
+ 
control socket, it gives an error message that helps the user to
+ 
understand the problem.
+
+ 
_Justification:_ This helps users deal with problems themselves and
+ 
reduces the support burden on the Radicle project.
+
+ 
_Stakeholder:_ Lars.
+
+ 
~~~scenario
+ 
given a directory homedir
+ 
when I run env HOME=homedir RAD_PASSPHRASE= rad auth --alias brokertest
+
+ 
given an installed ci-broker
+ 
given file broker.yaml
+ 
when I run sed -i 's/"auto"/false/' homedir/.radicle/config.json
+ 
when I try to run env HOME=homedir  RAD_SOCKET=xyzzy.sock ci-broker broker.yaml
+ 
then command fails
+ 
then stderr contains "ERROR: node control socket does not exist: xyzzy.sock"
+ 
~~~
+
+ 
## Gives helpful error message if it doesn't understand its configuration file
+
+ 
_Requirement:_ If the CI broker is given a configuration file that it
+ 
can't understand, it gives an error message that explains the problem
+ 
to the user.
+
+ 
_Justification:_ This helps users deal with problems themselves and
+ 
reduces the support burden on the Radicle project.
+
+ 
_Stakeholder:_ Lars.
+
+ 
_Comment:_ This is a very basic scenario. Error handling is by nature
+ 
a thing that can always be made better. We can later add more
+ 
scenarios if we tighten the acceptance criteria.
+
+ 
~~~scenario
+ 
given a directory homedir
+ 
when I run env HOME=homedir RAD_PASSPHRASE= rad auth --alias brokertest
+
+ 
given an installed ci-broker
+ 
given file broker.yaml
+ 
given file not-yaml.yaml
+ 
when I run sed -i 's/"auto"/false/' homedir/.radicle/config.json
+ 
when I try to run env HOME=homedir ci-broker not-yaml.yaml
+ 
then command fails
+ 
then stderr contains "ERROR: failed to parse configuration file as YAML: not-yaml.yaml"
+ 
~~~
+
+
+ 
~~~{#not-yaml.yaml .file}
+ 
This file is not YAML.
+ 
~~~
+
+ 
## Stops if the node connection breaks
+
+ 
_Requirement:_ If the connection to the Radicle node, via its control
+ 
socket, breaks, the CI broker terminates with a message saying why.
+
+ 
_Justification:_ The CI broker can either keep running and trying to
+ 
re-connect, or it can terminate. Either is workable. However, it's a
+ 
simpler design and less code to terminate and allow re-starting to be
+ 
handled by a dedicated system, such as `systemd`.
+
+ 
_Stakeholder:_ Lars.
+
+ 
~~~scenario
+ 
given a directory homedir
+ 
when I run env HOME=homedir RAD_PASSPHRASE= rad auth --alias brokertest
+
+ 
given an installed ci-broker
+ 
given an installed synthetic-events
+ 
when I run synthetic-events synt.sock
+ 
given file broker.yaml
+ 
when I run sed -i 's/"auto"/false/' homedir/.radicle/config.json
+ 
when I try to run env HOME=homedir RAD_SOCKET=synt.sock ci-broker broker.yaml
+ 
then command fails
+ 
then stderr contains "connection to the node control socket broke"
+ 
~~~
+
+
+ 
## Shuts down when requested
+
+ 
_Requirement:_ The test suite can request the CI broker to shut down
+ 
cleanly, and it doesn't result in an error.
+
+ 
_Justification:_ In the integration test suite, we need to start and
+ 
stop the CI broker many times. We need to easily detect errors.
+
+ 
_Stakeholder:_ Lars.
+
+ 
We use a special magic fake node event to signal shutdown: a
+ 
`RefsFetched` event with a skipped update for a ref "`shutdown`" and
+ 
an object id of all zeros. This should be sufficiently impossible to
+ 
happen in real life.
+
+ 
~~~scenario
+ 
given a directory homedir
+ 
when I run env HOME=homedir RAD_PASSPHRASE= rad auth --alias brokertest
+
+ 
given an installed ci-broker
+ 
given an installed synthetic-events
+ 
given file shutdown.json
+ 
given file broker.yaml
+ 
when I run synthetic-events synt.sock shutdown.json
+ 
when I run sed -i 's/"auto"/false/' homedir/.radicle/config.json
+ 
when I run sed -i 's/"auto"/false/' homedir/.radicle/config.json
+ 
when I try to run env RUST_LOG=trace HOME=homedir RAD_SOCKET=synt.sock ci-broker broker.yaml
+ 
then command is successful
+ 
~~~
+
+ 
~~~{#shutdown.json .file .json}
+ 
{
+ 
  "type": "refsFetched",
+ 
  "remote": "z6MkgEMYod7Hxfy9qCvDv5hYHkZ4ciWmLFgfvm3Wn1b2w2FV",
+ 
  "rid": "rad:zwTxygwuz5LDGBq255RA2CbNGrz8",
+ 
  "updated": [
+ 
    {
+ 
      "skipped": {
+ 
        "name": "shutdown",
+ 
        "oid": "0000000000000000000000000000000000000000"
+ 
      }
+ 
    }
+ 
  ]
+ 
}
+ 
~~~
+
+
+ 
# Acceptance criteria for test tooling
+
+ 
The event synthesizer is a helper to feed the CI broker node events in
+ 
a controlled fashion.
+
+ 
## Dummy adapter runs successfully
+
+ 
_Requirement:_ The dummy adapter (in embedded file `dummy.sh`) runs
+ 
successfully.
+
+ 
_Justification:_ Test scenarios using the dummy adapter need to be
+ 
able to rely that it works.
+
+ 
_Stakeholder:_ Lars
+
+ 
~~~scenario
+ 
given file dummy.sh
+ 
when I run chmod +x dummy.sh
+ 
when I try to run ./dummy.sh
+ 
then command is successful
+ 
~~~
+
+ 
## Event synthesizer terminates after first connection
+
+ 
_Requirement:_ The event synthesizer runs in the background, but
+ 
terminates after the first connection.
+
+ 
_Justification:_ This is needed so that it can be invoked in Subplot
+ 
scenarios.
+
+ 
_Stakeholder:_ Lars.
+
+ 
The following scenario may only work on Linux, as it's using `pgrep`
+ 
and `nc` and those may not be portable. If so, this may need to be
+ 
changed for other platforms.
+
+ 
~~~scenario
+ 
given an installed synthetic-events
+
+ 
when I try to run pgrep -f synthetic-events
+ 
then command fails
+
+ 
when I run synthetic-events synt.sock
+ 
when I run pgrep -f synthetic-events
+ 
then command is successful
+
+ 
when I run nc -U synt.sock
+ 
then command is successful
+
+ 
when I try to run pgrep -f synthetic-events
+ 
then command fails
+ 
~~~
added ci-broker.subplot
@@ -0,0 +1,14 @@
+ 
title: Radicle CI broker
+ 
subtitle: Acceptance criteria
+ 
authors:
+ 
  - The Radicle Project
+ 
  - Lars Wirzenius
+ 
markdowns:
+ 
  - ci-broker.md
+ 
bindings:
+ 
  - ci-broker.yaml
+ 
  - lib/files.yaml
+ 
  - lib/runcmd.yaml
+ 
impls:
+ 
  rust:
+ 
    - src/subplot.rs
added ci-broker.yaml
@@ -0,0 +1,9 @@
+ 
- given: "an installed ci-broker"
+ 
  impl:
+ 
    rust:
+ 
      function: install_ci_broker
+
+ 
- given: "an installed synthetic-events"
+ 
  impl:
+ 
    rust:
+ 
      function: install_synthetic_events
added src/subplot.rs
@@ -0,0 +1,48 @@
+ 
// Implementations of Subplot scenario steps for the CI broker.
+
+ 
use std::path::{Path, PathBuf};
+
+ 
use subplotlib::steplibrary::runcmd::Runcmd;
+
+ 
#[derive(Debug, Default)]
+ 
struct SubplotContext {}
+
+ 
impl ContextElement for SubplotContext {}
+
+ 
#[step]
+ 
#[context(SubplotContext)]
+ 
#[context(Runcmd)]
+ 
fn install_ci_broker(context: &ScenarioContext) {
+ 
    let target_path = bindir();
+ 
    assert!(target_path.join("ci-broker").exists());
+ 
    context.with_mut(
+ 
        |context: &mut Runcmd| {
+ 
            context.prepend_to_path(target_path);
+ 
            Ok(())
+ 
        },
+ 
        false,
+ 
    )?;
+ 
}
+
+ 
#[step]
+ 
#[context(SubplotContext)]
+ 
#[context(Runcmd)]
+ 
fn install_synthetic_events(context: &ScenarioContext) {
+ 
    let target_path = bindir();
+ 
    assert!(target_path.join("synthetic-events").exists());
+ 
    context.with_mut(
+ 
        |context: &mut Runcmd| {
+ 
            context.prepend_to_path(target_path);
+ 
            Ok(())
+ 
        },
+ 
        false,
+ 
    )?;
+ 
}
+
+ 
fn bindir() -> PathBuf {
+ 
    if let Ok(target) = std::env::var("CARGO_TARGET_DIR") {
+ 
        Path::new(&target).join("debug")
+ 
    } else {
+ 
        PathBuf::from("target/debug")
+ 
    }
+ 
}
added tests/subplot.rs
@@ -0,0 +1 @@
+ 
include!(concat!(env!("OUT_DIR"), "/ci-broker.rs"));