Skip to content

Make dev OCI tests required #754

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
epompeii wants to merge 1 commit into
base: devel
Choose a base branch
from
Open

Make dev OCI tests required #754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions .github/workflows/deploy.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,6 @@ jobs:
env:
RUST_BACKTRACE: full
run: cargo test-api smoke dev
- name: Run OCI Conformance Test
continue-on-error: true
timeout-minutes: 5
run: |
cargo test-api oci \
--url https://dev.registry.bencher.dev/
- name: OCI Conformance Test Summary
if: always()
run: |
Expand Down
33 changes: 27 additions & 6 deletions tasks/test_api/src/task/test/smoke_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ use std::{
};

use bencher_json::{
BENCHER_API_PORT, DEV_BENCHER_API_URL, JsonApiVersion, Jwt, LOCALHOST_BENCHER_API_URL,
PROD_BENCHER_API_URL, TEST_BENCHER_API_URL, Url,
BENCHER_API_PORT, DEV_BENCHER_API_URL, DEV_BENCHER_REGISTRY_URL, JsonApiVersion, Jwt,
LOCALHOST_BENCHER_API_URL, LOCALHOST_BENCHER_REGISTRY_URL, PROD_BENCHER_API_URL,
PROD_BENCHER_REGISTRY_URL, TEST_BENCHER_API_URL, TEST_BENCHER_REGISTRY_URL, Url,
};

use crate::{
Expand Down Expand Up @@ -82,12 +83,14 @@ impl SmokeTest {
};

let api_url = self.environment.as_url();
let registry_url = self.environment.as_registry_url();
test_api_version(&api_url)?;

match self.environment {
Environment::Ci => {
test(
&api_url,
&registry_url,
MockSetup::SelfHosted {
examples: false,
runner: true,
Expand All @@ -98,6 +101,7 @@ impl SmokeTest {
Environment::Localhost => {
test(
&api_url,
&registry_url,
MockSetup::SelfHosted {
examples: true,
runner: false,
Expand All @@ -106,7 +110,7 @@ impl SmokeTest {
kill_child(child)?;
},
Environment::Docker => bencher_down()?,
Environment::Dev => test(&api_url, MockSetup::BencherCloud)?,
Environment::Dev => test(&api_url, &registry_url, MockSetup::BencherCloud)?,
Environment::Test | Environment::Prod => {},
}

Expand All @@ -124,6 +128,16 @@ impl Environment {
}
.into()
}

fn as_registry_url(self) -> Url {
match self {
Self::Ci | Self::Localhost | Self::Docker => LOCALHOST_BENCHER_REGISTRY_URL.clone(),
Self::Dev => DEV_BENCHER_REGISTRY_URL.clone(),
Self::Test => TEST_BENCHER_REGISTRY_URL.clone(),
Self::Prod => PROD_BENCHER_REGISTRY_URL.clone(),
}
.into()
}
}

fn api_run() -> anyhow::Result<Child> {
Expand Down Expand Up @@ -205,7 +219,7 @@ enum MockSetup {
SelfHosted { examples: bool, runner: bool },
}

fn test(api_url: &Url, mock_setup: MockSetup) -> anyhow::Result<()> {
fn test(api_url: &Url, registry_url: &Url, mock_setup: MockSetup) -> anyhow::Result<()> {
match mock_setup {
MockSetup::BencherCloud => {
let task = TaskSeedTest {
Expand All @@ -215,9 +229,16 @@ fn test(api_url: &Url, mock_setup: MockSetup) -> anyhow::Result<()> {
};
SeedTest::try_from(task)?.exec()?;

// Run runner smoke test (requires Docker + KVM for the runner daemon)
#[cfg(feature = "plus")]
{
// Run OCI conformance tests
let oci = Oci::try_from(TaskOci {
url: Some(registry_url.clone()),
..Default::default()
})?;
oci.exec()?;

// Run runner smoke test (requires Docker + KVM for the runner daemon)
let runner_test = runner::RunnerTest::try_from(TaskRunner {
url: Some(api_url.clone()),
with_daemon: true,
Expand All @@ -239,7 +260,7 @@ fn test(api_url: &Url, mock_setup: MockSetup) -> anyhow::Result<()> {
{
// Run OCI conformance tests
let oci = Oci::try_from(TaskOci {
url: Some(api_url.clone()),
url: Some(registry_url.clone()),
..Default::default()
})?;
oci.exec()?;
Expand Down
Loading