diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile index 1a2d0da2..315fc91c 100644 --- a/.buildkite/Dockerfile +++ b/.buildkite/Dockerfile @@ -12,7 +12,7 @@ RUN gem install rspec cucumber base64 RUN gem install bigdecimal -v 3.2.0 RUN yarn global add jest RUN pip install pytest -RUN pip install buildkite-test-collector==0.2.0 +RUN pip install buildkite-test-collector>=1.3.0 RUN curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash -s -- --bin-dir /usr/local/bin # Install curl, download bktec binary, make it executable, place it, and cleanup diff --git a/.tool-versions b/.tool-versions index 178fa14e..1c4bb899 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,4 @@ golang 1.25.2 nodejs 24.9.0 python 3.13.2 +uv 0.9.26 diff --git a/bin/setup b/bin/setup index 7903931f..09c5c195 100755 --- a/bin/setup +++ b/bin/setup @@ -9,14 +9,14 @@ go install github.com/pact-foundation/pact-go/v2 go install gotest.tools/gotestsum@v1.8.0 # Check if asdf is installed and being used for Go -if command -v asdf &> /dev/null && asdf current golang &> /dev/null; then +if command -v asdf &>/dev/null && asdf current golang &>/dev/null; then echo "🔄 Reshimming asdf golang..." asdf reshim golang fi # download and install the required libraries. # TODO if pact-go check return non- zero then install it -if ! pact-go check &> /dev/null; then +if ! pact-go check &>/dev/null; then echo "🔄 Installing pact-go dependencies..." sudo pact-go -l DEBUG install else @@ -28,8 +28,7 @@ echo "🛠️ Installing dependencies for sample projects..." pushd ./internal/runner/testdata || exit 1 # if yarn is available, use it to install dependencies # otherwise, use npm -if command -v yarn &> /dev/null -then +if command -v yarn &>/dev/null; then yarn install else npm install @@ -68,7 +67,7 @@ else python -m venv .venv && source .venv/bin/activate fi pip install pytest -pip install buildkite-test-collector==0.2.0 +pip install "buildkite-test-collector>=1.3.0" curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash echo "💖 Everything is fantastic!" diff --git a/cli.go b/cli.go index 5b42a74c..df654687 100644 --- a/cli.go +++ b/cli.go @@ -115,12 +115,19 @@ var filesFlag = &cli.StringFlag{ Sources: cli.EnvVars("BUILDKITE_TEST_ENGINE_FILES"), } -var testCommandFlag = &cli.StringFlag{ - Name: "test-command", +var tagFiltersFlag = &cli.StringFlag{ + Name: "tag-filters", Category: "TEST RUNNER", - Usage: "Test command", - Sources: cli.EnvVars("BUILDKITE_TEST_ENGINE_TEST_CMD"), - Destination: &cfg.TestCommand, + Usage: "Tag filters to apply when selecting tests to run (currently only Pytest is supported)", + Sources: cli.EnvVars("BUILDKITE_TEST_ENGINE_TAG_FILTERS"), + Destination: &cfg.TagFilters, +} + +var testCommandFlag = &cli.StringFlag{ + Name: "test-command", + Category: "TEST RUNNER", + Usage: "Test command", + Sources: cli.EnvVars("BUILDKITE_TEST_ENGINE_TEST_CMD"), } var testFilePatternFlag = &cli.StringFlag{ @@ -267,6 +274,7 @@ var cliCommand = &cli.Command{ Action: run, Flags: []cli.Flag{ filesFlag, + tagFiltersFlag, planIdentifierFlag, // Build Environment Flags organizationSlugFlag, @@ -304,6 +312,7 @@ var cliCommand = &cli.Command{ // we will remove these in future iterations. filesFlag, + tagFiltersFlag, // Dynamic Parallelism Flags maxParallelismFlag, targetTimeFlag, diff --git a/go.mod b/go.mod index 2faf6250..844ce770 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/buildkite/test-engine-client -go 1.24 +go 1.24.0 toolchain go1.24.1 @@ -16,6 +16,7 @@ require ( github.com/pact-foundation/pact-go/v2 v2.4.2 github.com/stretchr/testify v1.11.1 github.com/urfave/cli/v3 v3.6.2 + golang.org/x/mod v0.32.0 ) require ( diff --git a/go.sum b/go.sum index dbfd0a4c..6d5a7e12 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= diff --git a/internal/api/filter_tests.go b/internal/api/filter_tests.go index f430bca6..10c95e7f 100644 --- a/internal/api/filter_tests.go +++ b/internal/api/filter_tests.go @@ -22,8 +22,15 @@ type FilteredTestResponse struct { Tests []FilteredTest `json:"tests"` } -// FilterTests filters tests from the server. It returns a list of tests that need to be split by example. -// Currently, it only filters tests that are slow. +// FilterTests fetches test files from the server. It returns a list of test files that +// need to be split by example. +// +// Currently, it only fetches tests file that are slow and test files that have tests +// marked for skipping. +// +// The splitByExample flag is passed through to the server, which is false will only +// return test files that contain skipped tests, while true will also return slow test +// files. func (c Client) FilterTests(ctx context.Context, suiteSlug string, params FilterTestsParams) ([]FilteredTest, error) { url := fmt.Sprintf("%s/v2/analytics/organizations/%s/suites/%s/test_plan/filter_tests", c.ServerBaseUrl, c.OrganizationSlug, suiteSlug) @@ -33,7 +40,6 @@ func (c Client) FilterTests(ctx context.Context, suiteSlug string, params Filter URL: url, Body: params, }, &response) - if err != nil { return []FilteredTest{}, err } diff --git a/internal/command/request_param.go b/internal/command/request_param.go index c01bf213..210eec84 100644 --- a/internal/command/request_param.go +++ b/internal/command/request_param.go @@ -11,8 +11,13 @@ import ( ) // createRequestParam generates the parameters needed for a test plan request. -// For runners other than "rspec", it constructs the test plan parameters with all test files. -// For the "rspec" runner, it filters the test files through the Test Engine API and splits the filtered files into examples. +// +// For the Rspec, Cucumber and Pytest runner, it fetches test files through the Test Engine API +// that are slow or contain skipped tests. These files are then split into examples +// The remaining files are sent as is. +// +// If tag filtering is enabled, all files are split into examples to support filtering. +// Currently only the Pytest runner supports tag filtering. func createRequestParam(ctx context.Context, cfg *config.Config, files []string, client api.Client, runner TestRunner) (api.TestPlanParams, error) { testFiles := []plan.TestCase{} for _, file := range files { @@ -50,10 +55,21 @@ func createRequestParam(ctx context.Context, cfg *config.Config, files []string, debug.Println("Splitting by example") } - // The SplitByExample flag indicates whether to filter slow files for splitting by example. - // Regardless of the flag's state, the API will still filter other files that need to be split by example, such as those containing skipped tests. - // Therefore, we must filter and split files even when SplitByExample is disabled. - testParams, err := filterAndSplitFiles(ctx, cfg, client, testFiles, runner) + var testParams api.TestPlanParamsTest + var err error + + // If tag filtering is enabled, we must split all files to allow to enable filtering. + // Tag filtering is currently only supported for pytest. + if cfg.TagFilters != "" && runner.Name() == "pytest" { + testParams, err = splitAllFiles(testFiles, runner) + } else { + // The SplitByExample flag indicates whether to split slow files into examples. + // Regardless of the flag's state, the API will still return other test files that need to + // be split by example, such as those containing skipped tests. + // Therefore, we must fetch and split files even when SplitByExample is disabled. + testParams, err = filterAndSplitFiles(ctx, cfg, client, testFiles, runner) + } + if err != nil { return api.TestPlanParams{}, err } @@ -69,6 +85,26 @@ func createRequestParam(ctx context.Context, cfg *config.Config, files []string, }, nil } +// Splits all the test files into examples to support tag filtering. +func splitAllFiles(files []plan.TestCase, runner TestRunner) (api.TestPlanParamsTest, error) { + debug.Printf("Splitting all %d files", len(files)) + filePaths := make([]string, 0, len(files)) + for _, file := range files { + filePaths = append(filePaths, file.Path) + } + + examples, err := runner.GetExamples(filePaths) + if err != nil { + return api.TestPlanParamsTest{}, fmt.Errorf("get examples: %w", err) + } + + debug.Printf("Got %d examples from all files", len(examples)) + + return api.TestPlanParamsTest{ + Examples: examples, + }, nil +} + // filterAndSplitFiles filters the test files through the Test Engine API and splits the filtered files into examples. // It returns the test plan parameters with the examples from the filtered files and the remaining files. // An error is returned if there is a failure in any of the process. diff --git a/internal/command/run_test.go b/internal/command/run_test.go index 31bf851c..5e177908 100644 --- a/internal/command/run_test.go +++ b/internal/command/run_test.go @@ -693,7 +693,6 @@ func TestCreateRequestParams(t *testing.T) { TestCommand: "rspec", }, }) - if err != nil { t.Errorf("createRequestParam() error = %v", err) } @@ -778,7 +777,6 @@ func TestCreateRequestParams_NonRSpec(t *testing.T) { } got, err := createRequestParam(context.Background(), &cfg, files, *client, r) - if err != nil { t.Errorf("createRequestParam() error = %v", err) } @@ -838,7 +836,6 @@ func TestCreateRequestParams_PytestPants(t *testing.T) { } got, err := createRequestParam(context.Background(), &cfg, files, *client, runner) - if err != nil { t.Errorf("createRequestParam() error = %v", err) } @@ -935,7 +932,6 @@ func TestCreateRequestParams_NoFilteredFiles(t *testing.T) { TestCommand: "rspec", }, }) - if err != nil { t.Errorf("createRequestParam() error = %v", err) } @@ -962,6 +958,122 @@ func TestCreateRequestParams_NoFilteredFiles(t *testing.T) { } } +func TestCreateRequestParams_WithTagFilters(t *testing.T) { + cfg := config.Config{ + OrganizationSlug: "my-org", + SuiteSlug: "my-suite", + Identifier: "identifier", + Parallelism: 2, + Branch: "main", + TestRunner: "pytest", + TagFilters: "team:frontend", + } + + client := api.NewClient(api.ClientConfig{ + ServerBaseUrl: "example.com", + }) + + files := []string{ + "../runner/testdata/pytest/failed_test.py", + "../runner/testdata/pytest/test_sample.py", + "../runner/testdata/pytest/spells/test_expelliarmus.py", + } + + got, err := createRequestParam(context.Background(), &cfg, files, *client, runner.Pytest{ + RunnerConfig: runner.RunnerConfig{ + TestCommand: "pytest", + TagFilters: "team:frontend", + }, + }) + if err != nil { + t.Errorf("createRequestParam() error = %v", err) + } + + want := api.TestPlanParams{ + Identifier: "identifier", + Parallelism: 2, + Branch: "main", + Runner: "pytest", + Tests: api.TestPlanParamsTest{ + Examples: []plan.TestCase{ + { + Format: "example", + Identifier: "runner/testdata/pytest/test_sample.py::test_happy", + Name: "test_happy", + Path: "runner/testdata/pytest/test_sample.py::test_happy", + Scope: "runner/testdata/pytest/test_sample.py", + }, + { + Format: "example", + Identifier: "runner/testdata/pytest/spells/test_expelliarmus.py::TestExpelliarmus::test_knocks_wand_out", + Name: "test_knocks_wand_out", + Path: "runner/testdata/pytest/spells/test_expelliarmus.py::TestExpelliarmus::test_knocks_wand_out", + Scope: "runner/testdata/pytest/spells/test_expelliarmus.py::TestExpelliarmus", + }, + }, + }, + } + + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("createRequestParam() diff (-got +want):\n%s", diff) + } +} + +func TestCreateRequestParams_WithTagFilters_NonPytest(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, ` +{ + "tests": [] +}`) + })) + defer svr.Close() + + cfg := config.Config{ + OrganizationSlug: "my-org", + SuiteSlug: "my-suite", + Identifier: "identifier", + Parallelism: 2, + Branch: "main", + TestRunner: "rspec", + TagFilters: "team:frontend", + } + + client := api.NewClient(api.ClientConfig{ + ServerBaseUrl: svr.URL, + }) + + files := []string{ + "testdata/rspec/spec/fruits/apple_spec.rb", + "testdata/rspec/spec/fruits/banana_spec.rb", + } + + got, err := createRequestParam(context.Background(), &cfg, files, *client, runner.Rspec{ + RunnerConfig: runner.RunnerConfig{ + TestCommand: "rspec", + }, + }) + if err != nil { + t.Errorf("createRequestParam() error = %v", err) + } + + want := api.TestPlanParams{ + Identifier: "identifier", + Parallelism: 2, + Branch: "main", + Runner: "rspec", + Tests: api.TestPlanParamsTest{ + Files: []plan.TestCase{ + {Path: "testdata/rspec/spec/fruits/apple_spec.rb"}, + {Path: "testdata/rspec/spec/fruits/banana_spec.rb"}, + }, + }, + } + + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("createRequestParam() diff (-got +want):\n%s", diff) + } +} + func TestSendMetadata(t *testing.T) { originalVersion := version.Version version.Version = "0.1.0" @@ -1019,7 +1131,6 @@ func TestSendMetadata(t *testing.T) { } else { w.WriteHeader(http.StatusOK) } - })) defer svr.Close() @@ -1070,7 +1181,6 @@ func TestRunTestsWithRetry_NoTestCases_Success(t *testing.T) { failOnNoTests := false testResult, err := runTestsWithRetry(testRunner, &testCases, maxRetries, []plan.TestCase{}, &timeline, true, failOnNoTests) - if err != nil { t.Errorf("runTestsWithRetry(...) error = %v, want nil", err) } diff --git a/internal/config/config.go b/internal/config/config.go index a0a4bcfe..7a99bae7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -52,6 +52,8 @@ type Config struct { DebugEnabled bool `json:"BUILDKITE_TEST_ENGINE_DEBUG_ENABLED"` // FailOnNoTests causes the client to exit with an error if no tests are assigned to the node FailOnNoTests bool `json:"BUILDKITE_TEST_ENGINE_FAIL_ON_NO_TESTS"` + // TagFilters filters test examples by execution tags. + TagFilters string `json:"BUILDKITE_TEST_ENGINE_TAG_FILTERS"` // errs is a map of environment variables name and the validation errors associated with them. errs InvalidConfigError } diff --git a/internal/config/validate.go b/internal/config/validate.go index 2a028b1e..d68e1f3d 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -68,6 +68,13 @@ func (c *Config) validate() error { } } + if c.TagFilters != "" && c.TestRunner != "pytest" { + c.errs.appendFieldError( + "BUILDKITE_TEST_ENGINE_TAG_FILTERS", + "tag filtering is only supported for the pytest test runner", + ) + } + if len(c.errs) > 0 { return c.errs } diff --git a/internal/config/validate_test.go b/internal/config/validate_test.go index 219f8393..f9910d88 100644 --- a/internal/config/validate_test.go +++ b/internal/config/validate_test.go @@ -344,3 +344,39 @@ func ValidateForPlan_SkipsParallelismAndNodeIndexValidation(t *testing.T) { t.Errorf("config.validate() err = %v, want nil", err) } } + +func TestConfigValidate_TagFiltersOnlyWorksWithPytest(t *testing.T) { + t.Run("TagFilters with pytest runner should be valid", func(t *testing.T) { + c := createConfig() + c.TestRunner = "pytest" + c.TagFilters = "speed=slow" + + err := c.validate() + if err != nil { + t.Errorf("config.validate() error = %v, want nil", err) + } + }) + + t.Run("TagFilters with non-pytest runner should fail", func(t *testing.T) { + c := createConfig() + c.TestRunner = "rspec" + c.TagFilters = "speed=slow" + + err := c.validate() + + var invConfigError InvalidConfigError + if !errors.As(err, &invConfigError) { + t.Errorf("config.validate() error = %v, want InvalidConfigError", err) + return + } + + if len(invConfigError["BUILDKITE_TEST_ENGINE_TAG_FILTERS"]) != 1 { + t.Errorf("config.validate() error for BUILDKITE_TEST_ENGINE_TAG_FILTERS length = %d, want 1", len(invConfigError["BUILDKITE_TEST_ENGINE_TAG_FILTERS"])) + } + + expectedMsg := "tag filtering is only supported for the pytest test runner" + if invConfigError["BUILDKITE_TEST_ENGINE_TAG_FILTERS"][0].Error() != expectedMsg { + t.Errorf("config.validate() error message = %q, want %q", invConfigError["BUILDKITE_TEST_ENGINE_TAG_FILTERS"][0].Error(), expectedMsg) + } + }) +} diff --git a/internal/runner/detector.go b/internal/runner/detector.go index b56eaf43..3d97c40e 100644 --- a/internal/runner/detector.go +++ b/internal/runner/detector.go @@ -13,6 +13,7 @@ type RunnerConfig struct { TestFilePattern string TestFileExcludePattern string RetryTestCommand string + TagFilters string // ResultPath is used internally so bktec can read result from Test Runner. // User typically don't need to worry about setting this except in in RSpec and playwright. // In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set. @@ -27,13 +28,14 @@ type TestRunner interface { } func DetectRunner(cfg *config.Config) (TestRunner, error) { - var runnerConfig = RunnerConfig{ + runnerConfig := RunnerConfig{ TestRunner: cfg.TestRunner, TestCommand: cfg.TestCommand, TestFilePattern: cfg.TestFilePattern, TestFileExcludePattern: cfg.TestFileExcludePattern, RetryTestCommand: cfg.RetryCommand, ResultPath: cfg.ResultPath, + TagFilters: cfg.TagFilters, } switch testRunner := cfg.TestRunner; testRunner { diff --git a/internal/runner/pytest.go b/internal/runner/pytest.go index 2b99957f..fc7ce97e 100644 --- a/internal/runner/pytest.go +++ b/internal/runner/pytest.go @@ -11,6 +11,7 @@ import ( "github.com/buildkite/test-engine-client/internal/debug" "github.com/buildkite/test-engine-client/internal/plan" "github.com/kballard/go-shellquote" + "golang.org/x/mod/semver" ) type Pytest struct { @@ -28,6 +29,15 @@ func NewPytest(c RunnerConfig) Pytest { os.Exit(1) } + // Ensure buildkite-test-collector version is >1.2.0 for --tag-filters support + if c.TagFilters != "" { + if err := checkBuildkiteTestCollectorVersion("1.2.0"); err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + fmt.Fprintln(os.Stderr, "Please upgrade with: pip install --upgrade buildkite-test-collector") + os.Exit(1) + } + } + if c.TestCommand == "" { c.TestCommand = "pytest {{testExamples}} --json={{resultPath}}" } @@ -84,7 +94,6 @@ func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) er } for _, test := range tests { - result.RecordTestResult(plan.TestCase{ Identifier: test.Id, Format: plan.TestCaseFormatExample, @@ -117,12 +126,23 @@ func (p Pytest) GetFiles() ([]string, error) { // GetExamples returns an array of test examples within the given files. // It uses `pytest --collect-only -q` to enumerate individual tests. +// +// --tag-filters can be used to filter tests by markers if specified. +// e.g. --tag-fitlers team:frontend matches markers: +// with @pytest.mark.execution_tag('team', 'frontend') +// +// The --tag-filters feature also assumes Python Test Collector plugin +// version >1.2.0 is installed. func (p Pytest) GetExamples(files []string) ([]plan.TestCase, error) { if len(files) == 0 { return []plan.TestCase{}, nil } - args := append([]string{"--collect-only", "-q"}, files...) + args := []string{"--collect-only", "-q"} + if p.TagFilters != "" { + args = append(args, "--tag-filters", p.TagFilters) + } + args = append(args, files...) cmd := exec.Command("pytest", args...) output, err := cmd.Output() @@ -200,7 +220,6 @@ func (p Pytest) commandNameAndArgs(cmd string, testCases []string) (string, []st cmd = strings.Replace(cmd, "{{resultPath}}", p.ResultPath, 1) args, err := shellquote.Split(cmd) - if err != nil { return "", []string{}, err } @@ -216,6 +235,47 @@ func getRandomTempFilename() string { return filepath.Join(tempDir, "pytest-results.json") } +// getPythonPackageVersion retrieves the version of a Python package using importlib.metadata. +// The pkgName should use hyphens (e.g., "buildkite-test-collector") as that's the package name in metadata. +func getPythonPackageVersion(pkgName string) (string, error) { + pythonCmd := exec.Command("python", "-c", "import importlib.metadata, sys; print(importlib.metadata.version(sys.argv[1]))", pkgName) + output, err := pythonCmd.Output() + if err != nil { + return "", fmt.Errorf("could not determine %s version: %w", pkgName, err) + } + + return strings.TrimSpace(string(output)), nil +} + +// checkBuildkiteTestCollectorVersion verifies that the installed buildkite-test-collector +// version is greater than the specified required version. +func checkBuildkiteTestCollectorVersion(requiredVersion string) error { + installedVersionStr, err := getPythonPackageVersion("buildkite-test-collector") + if err != nil { + return err + } + + // semver package requires versions to be prefixed with "v" + installedVersionCanonical := "v" + installedVersionStr + requiredVersionCanonical := "v" + requiredVersion + + if !semver.IsValid(installedVersionCanonical) { + return fmt.Errorf("could not parse installed buildkite-test-collector version %q", installedVersionStr) + } + + if !semver.IsValid(requiredVersionCanonical) { + return fmt.Errorf("could not parse required version %q", requiredVersion) + } + + // semver.Compare returns -1 if v1 < v2, 0 if v1 == v2, 1 if v1 > v2 + // We want installed > required, so compare should return 1 + if semver.Compare(installedVersionCanonical, requiredVersionCanonical) <= 0 { + return fmt.Errorf("buildkite-test-collector version %s is installed, but version >%s is required for --tag-filters support", installedVersionStr, requiredVersion) + } + + return nil +} + func checkPythonPackageInstalled(pkgName string) bool { // This is the most reliable way I can find. Hopefully it should work regardless of if user uses pip, poetry or uv pythonCmd := exec.Command("python", "-c", "import importlib.util, sys; print(importlib.util.find_spec(sys.argv[1]) is not None)", pkgName) diff --git a/internal/runner/pytest_test.go b/internal/runner/pytest_test.go index 6ee40d00..3cfece93 100644 --- a/internal/runner/pytest_test.go +++ b/internal/runner/pytest_test.go @@ -20,7 +20,6 @@ func TestPytestRun(t *testing.T) { } result := NewRunResult([]plan.TestCase{}) err := pytest.Run(result, testCases, false) - if err != nil { t.Errorf("Pytest.Run(%q) error = %v", testCases, err) } @@ -44,7 +43,6 @@ func TestPytestRun_RetryCommand(t *testing.T) { result := NewRunResult([]plan.TestCase{}) err := pytest.Run(result, testCases, true) - if err != nil { t.Errorf("Pytest.Run(%q) error = %v", testCases, err) } @@ -62,7 +60,6 @@ func TestPytestRun_TestFailed(t *testing.T) { } result := NewRunResult([]plan.TestCase{}) err := pytest.Run(result, testCases, false) - if err != nil { t.Errorf("Pytest.Run(%q) error = %v", testCases, err) } @@ -273,7 +270,6 @@ func TestPytestGetExamples(t *testing.T) { pytest := NewPytest(RunnerConfig{}) files := []string{"spells/test_expelliarmus.py"} got, err := pytest.GetExamples(files) - if err != nil { t.Fatalf("Pytest.GetExamples(%q) error = %v", files, err) } @@ -303,7 +299,6 @@ func TestPytestGetExamples(t *testing.T) { func TestPytestGetExamples_EmptyFiles(t *testing.T) { pytest := NewPytest(RunnerConfig{}) got, err := pytest.GetExamples([]string{}) - if err != nil { t.Errorf("Pytest.GetExamples([]) error = %v", err) } @@ -312,6 +307,35 @@ func TestPytestGetExamples_EmptyFiles(t *testing.T) { } } +func TestPytestGetExamples_TagFilter(t *testing.T) { + changeCwd(t, "./testdata/pytest") + + pytest := NewPytest( + RunnerConfig{ + TagFilters: "team:frontend", + }, + ) + + files, _ := pytest.GetFiles() + + got, err := pytest.GetExamples(files) + if err != nil { + t.Fatalf("Pytest.GetExamples(%q) error = %v", files, err) + } + + if len(got) != 2 { + t.Fatalf("Pytest.GetExamples(%q) with tag filter 'team:frontend' returned %d tests, want 2", files, len(got)) + } + + if got[0].Name != "test_knocks_wand_out" { + t.Errorf("got[0].Name = %q, want %q", got[0].Name, "test_knocks_wand_out") + } + + if got[1].Name != "test_happy" { + t.Errorf("got[0].Name = %q, want %q", got[0].Name, "test_happy") + } +} + func TestParsePytestCollectOutput(t *testing.T) { output := `test_sample.py::test_happy test_auth.py::TestLogin::test_success diff --git a/internal/runner/testdata/pytest/spells/test_expelliarmus.py b/internal/runner/testdata/pytest/spells/test_expelliarmus.py index 0a5f30c1..a2179677 100644 --- a/internal/runner/testdata/pytest/spells/test_expelliarmus.py +++ b/internal/runner/testdata/pytest/spells/test_expelliarmus.py @@ -1,7 +1,10 @@ +import pytest + class TestExpelliarmus: + @pytest.mark.execution_tag("team", "backend") def test_disarms_opponent(self): assert True + @pytest.mark.execution_tag("team", "frontend") def test_knocks_wand_out(self): assert True - diff --git a/internal/runner/testdata/pytest/test_sample.py b/internal/runner/testdata/pytest/test_sample.py index 5d60784e..4273b9f2 100644 --- a/internal/runner/testdata/pytest/test_sample.py +++ b/internal/runner/testdata/pytest/test_sample.py @@ -1,2 +1,6 @@ +import pytest + +@pytest.mark.execution_tag("priority", "high") +@pytest.mark.execution_tag("team", "frontend") def test_happy(): assert 3 == 3