Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/branch-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,9 @@ jobs:
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
test-matrix: >-
[
{"scenario":"ubuntu-docker-rootful","testsuite":"conformance"},
{"scenario":"fedora-podman-rootful","testsuite":"conformance"},
{"scenario":"fedora-podman-rootless","testsuite":"conformance"}
{"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"},
{"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
{"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
]

# Run feature-specific integration tests:
Expand All @@ -242,8 +242,8 @@ jobs:
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
test-matrix: >-
[
{"scenario":"fedora-podman-rootful","testsuite":"provider-refresh"},
{"scenario":"fedora-podman-rootless","testsuite":"provider-refresh"}
{"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"provider-refresh"},
{"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"}
]

# Run driver-specific integration tests:
Expand Down
15 changes: 8 additions & 7 deletions .github/workflows/integration-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ on:
required: false
type: string
test-matrix:
description: JSON array of scenario and testsuite combinations
description: JSON array of environment, installer, and testsuite combinations
required: false
type: string
default: >-
[
{"scenario":"ubuntu-docker-rootful","testsuite":"conformance"},
{"scenario":"fedora-podman-rootful","testsuite":"conformance"},
{"scenario":"fedora-podman-rootless","testsuite":"conformance"}
{"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"},
{"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
{"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
]

permissions:
Expand All @@ -40,7 +40,7 @@ concurrency:

jobs:
integration:
name: Integration test (${{ inputs.category }}, ${{ matrix.scenario }}, ${{ matrix.testsuite }})
name: Integration test (${{ inputs.category }}, ${{ matrix.environment }}, ${{ matrix.installer }}, ${{ matrix.testsuite }})
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -80,6 +80,7 @@ jobs:

- name: Run test suite
env:
SCENARIO: ${{ matrix.scenario }}
ENVIRONMENT: ${{ matrix.environment }}
INSTALLER: ${{ matrix.installer }}
TESTSUITE: ${{ matrix.testsuite }}
run: nix run .#tmachine -- test "${SCENARIO}" "${TESTSUITE}"
run: nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}"
Comment thread
SDAChess marked this conversation as resolved.
8 changes: 4 additions & 4 deletions .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ on:
type: string
default: conformance
test-matrix:
description: JSON array of scenario and testsuite combinations
description: JSON array of environment, installer, and testsuite combinations
required: true
type: string
default: >-
[
{"scenario":"ubuntu-docker-rootful","testsuite":"conformance"},
{"scenario":"fedora-podman-rootful","testsuite":"conformance"},
{"scenario":"fedora-podman-rootless","testsuite":"conformance"}
{"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"},
{"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
{"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
]

permissions:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ jobs:
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
test-matrix: >-
[
{"scenario":"fedora-podman-rootful","testsuite":"provider-refresh"},
{"scenario":"fedora-podman-rootless","testsuite":"provider-refresh"}
{"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"provider-refresh"},
{"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"}
]

docker-e2e:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,8 @@ jobs:
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
test-matrix: >-
[
{"scenario":"fedora-podman-rootful","testsuite":"provider-refresh"},
{"scenario":"fedora-podman-rootless","testsuite":"provider-refresh"}
{"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"provider-refresh"},
{"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"provider-refresh"}
]

docker-e2e:
Expand Down
12 changes: 9 additions & 3 deletions architecture/build.md
Original file line number Diff line number Diff line change
Expand Up @@ -263,12 +263,18 @@ dependencies after installation. The Keycloak provider-refresh suite starts a
guest-local Keycloak realm and verifies a successful OAuth refresh followed by
revocation and the gateway's reauthorization-required recovery state.

The `tests/tmachine` setup and installation caches include a digest of the
Tmachine environments define the guest machine and runtime setup, while named
installers define how OpenShell is installed. This keeps the runtime mode
independent from binary or package installation and lets multiple installers
reuse the same prepared setup disk. The test command is
`tmachine test <environment> <installer> <testsuite>`.

The `tests/tmachine` setup and install caches include a digest of the
entire directory containing `ANSIBLE_CONFIG`, including local roles, task
includes, templates, inventory, and requirements. The digest uses sorted
relative paths, file contents, and executable permissions; source symlinks
are unsupported. Both keys also retain the ordered playbook paths and contents,
their base disk contents, and whether Galaxy is enabled; installation keys
their base disk contents, and whether Galaxy is enabled; install keys
include named binary inputs. The top-level `.roles` directory is excluded:
Galaxy release pins in `requirements.yaml` are treated as immutable, including
any transitive dependency pins. Cache misses with Galaxy enabled reinstall
Expand All @@ -279,7 +285,7 @@ with musl, and the gateway and supervisor with GNU. Image assembly stages
the gateway, sandbox, and supervisor as separate binaries for their respective
Dockerfiles. The helpers stage binaries under `artifacts/binaries` so local and
CI builds expose the same inputs to tmachine and image assembly. The Ubuntu
Docker and Fedora Podman scenarios import both local runtime images and
Docker and Fedora Podman environments import both local runtime images and
configure the gateway to use them.

## Python Wheel Packaging
Expand Down
56 changes: 17 additions & 39 deletions tests/config.nix
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ let
}
];

scenarios = [
environments = [
{
name = "ubuntu-docker-rootful";
machine = "ubuntu";
Expand All @@ -42,19 +42,6 @@ let
"ansible/playbooks/docker.yaml"
];
};
install = {
use_galaxy = false;
playbooks = [
"ansible/playbooks/openshell.yaml"
"ansible/playbooks/gateway.yaml"
];
inputs = {
openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
openshell_gateway_binary = "../artifacts/binaries/${gnuTarget}/openshell-gateway";
openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
};
};
}
{
name = "fedora-podman-rootful";
Expand All @@ -67,19 +54,6 @@ let
"ansible/playbooks/podman-rootful.yaml"
];
};
install = {
use_galaxy = false;
playbooks = [
"ansible/playbooks/openshell.yaml"
"ansible/playbooks/gateway.yaml"
];
inputs = {
openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
openshell_gateway_binary = "../artifacts/binaries/${gnuTarget}/openshell-gateway";
openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
};
};
}
{
name = "fedora-podman-rootless";
Expand All @@ -92,18 +66,22 @@ let
"ansible/playbooks/podman-rootless.yaml"
];
};
install = {
use_galaxy = false;
playbooks = [
"ansible/playbooks/openshell.yaml"
"ansible/playbooks/gateway.yaml"
];
inputs = {
openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
openshell_gateway_binary = "../artifacts/binaries/${gnuTarget}/openshell-gateway";
openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
};
}
];

installers = [
{
name = "binaries";
use_galaxy = false;
playbooks = [
"ansible/playbooks/openshell.yaml"
"ansible/playbooks/gateway.yaml"
];
inputs = {
openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
openshell_gateway_binary = "../artifacts/binaries/${gnuTarget}/openshell-gateway";
openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
};
}
];
Expand Down
9 changes: 5 additions & 4 deletions tests/tmachine/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ use serde::Deserialize;
#[derive(Clone, Deserialize)]
pub struct Config {
pub machines: Vec<Machine>,
pub scenarios: Vec<Scenario>,
pub environments: Vec<Environment>,
pub installers: Vec<Installer>,
pub testsuites: Vec<Testsuite>,
}

Expand All @@ -21,11 +22,10 @@ pub struct Machine {
}

#[derive(Clone, Deserialize)]
pub struct Scenario {
pub struct Environment {
pub name: String,
pub machine: String,
pub setup: Setup,
pub install: Install,
}

#[derive(Clone, Deserialize)]
Expand All @@ -35,7 +35,8 @@ pub struct Setup {
}

#[derive(Clone, Deserialize)]
pub struct Install {
pub struct Installer {
pub name: String,
pub use_galaxy: bool,
pub playbooks: Vec<PathBuf>,
pub inputs: BTreeMap<String, PathBuf>,
Expand Down
68 changes: 46 additions & 22 deletions tests/tmachine/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::path::PathBuf;

use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use config::{Config, Machine, Scenario};
use config::{Config, Environment, Installer, Machine};

mod ansible;
mod config;
Expand All @@ -23,9 +23,18 @@ struct Cli {

#[derive(Subcommand)]
enum Command {
Setup { scenario: String },
Install { scenario: String },
Test { scenario: String, testsuite: String },
Setup {
environment: String,
},
Install {
environment: String,
installer: String,
},
Test {
environment: String,
installer: String,
testsuite: String,
},
}

#[tokio::main(flavor = "current_thread")]
Expand All @@ -34,49 +43,64 @@ async fn main() -> Result<()> {
let config = Config::load(&cli.config)?;

match cli.command {
Command::Setup { scenario } => {
let (machine, scenario) = find_scenario(&config, &scenario)?;
qemu::setup(&machine, &scenario).await?;
Command::Setup { environment } => {
let (machine, environment) = find_environment(&config, &environment)?;
qemu::setup(&machine, &environment).await?;
}
Command::Install { scenario } => {
let (machine, scenario) = find_scenario(&config, &scenario)?;
qemu::install(&machine, &scenario).await?;
Command::Install {
environment,
installer,
} => {
let (machine, environment) = find_environment(&config, &environment)?;
let installer = find_installer(&config, &installer)?;
qemu::install(&machine, &environment, &installer).await?;
}
Command::Test {
scenario,
environment,
installer,
testsuite,
} => {
let (machine, scenario) = find_scenario(&config, &scenario)?;
let (machine, environment) = find_environment(&config, &environment)?;
let installer = find_installer(&config, &installer)?;
let testsuite = config
.testsuites
.iter()
.find(|candidate| candidate.name == testsuite)
.with_context(|| format!("testsuite {testsuite:?} is not defined"))?;
qemu::test(&machine, &scenario, testsuite).await?;
qemu::test(&machine, &environment, &installer, testsuite).await?;
}
}

Ok(())
}

fn find_scenario(config: &Config, name: &str) -> Result<(Machine, Scenario)> {
let scenario = config
.scenarios
fn find_installer(config: &Config, name: &str) -> Result<Installer> {
config
.installers
.iter()
.find(|installer| installer.name == name)
.with_context(|| format!("installer {name:?} is not defined"))
.cloned()
}

fn find_environment(config: &Config, name: &str) -> Result<(Machine, Environment)> {
let environment = config
.environments
.iter()
.find(|scenario| scenario.name == name)
.with_context(|| format!("scenario {name:?} is not defined"))?
.find(|environment| environment.name == name)
.with_context(|| format!("environment {name:?} is not defined"))?
.clone();
let machine = config
.machines
.iter()
.find(|machine| machine.name == scenario.machine)
.find(|machine| machine.name == environment.machine)
.with_context(|| {
format!(
"machine {:?} referenced by scenario {:?} is not defined",
scenario.machine, scenario.name
"machine {:?} referenced by environment {:?} is not defined",
environment.machine, environment.name
)
})?
.clone();

Ok((machine, scenario))
Ok((machine, environment))
}
Loading
Loading