From 39d3e63485164ba2f9d01bc0e0e0cdbcf3d809dc Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Sun, 24 May 2026 08:03:59 +0000
Subject: [PATCH 1/8] fix: add automated e2e testing using playground in CI/CD
Adds a PowerShell test runner (Run-E2ETests.ps1) that validates all three
ReferenceCop rule types (AssemblyName, ProjectTag, ProjectPath) plus a
valid-build scenario. Integrates into both PR and official build workflows.
New test projects: InternalLib, ToolsProject, ValidApp provide test fixtures
for the ProjectPath, ProjectTag, and valid-build scenarios respectively.
Fixes mfogliatto/ReferenceCop#43
---
.github/workflows/official-build.yaml | 8 +
.github/workflows/pr-build.yaml | 10 +-
playground/Run-E2ETests.ps1 | 214 ++++++++++++++++++
playground/TEST-SCENARIOS.md | 27 +++
.../TestProject/InternalLib/InternalHelper.cs | 6 +
.../InternalLib/InternalLib.csproj | 10 +
playground/TestProject/ToolsProject/Tool.cs | 6 +
.../ToolsProject/ToolsProject.csproj | 10 +
playground/TestProject/ValidApp/Program.cs | 1 +
.../TestProject/ValidApp/ValidApp.csproj | 15 ++
playground/playground.slnx | 3 +
11 files changed, 309 insertions(+), 1 deletion(-)
create mode 100644 playground/Run-E2ETests.ps1
create mode 100644 playground/TEST-SCENARIOS.md
create mode 100644 playground/TestProject/InternalLib/InternalHelper.cs
create mode 100644 playground/TestProject/InternalLib/InternalLib.csproj
create mode 100644 playground/TestProject/ToolsProject/Tool.cs
create mode 100644 playground/TestProject/ToolsProject/ToolsProject.csproj
create mode 100644 playground/TestProject/ValidApp/Program.cs
create mode 100644 playground/TestProject/ValidApp/ValidApp.csproj
diff --git a/.github/workflows/official-build.yaml b/.github/workflows/official-build.yaml
index c240aa1..5158e8c 100644
--- a/.github/workflows/official-build.yaml
+++ b/.github/workflows/official-build.yaml
@@ -35,6 +35,14 @@ jobs:
run: dotnet test --verbosity normal
working-directory: src/ReferenceCop.MSBuild.Tests
+ - name: Pack ReferenceCop for playground
+ run: dotnet pack -c Debug --nologo
+ working-directory: src/ReferenceCop.Package
+
+ - name: Run end-to-end tests
+ run: ./playground/Run-E2ETests.ps1
+ shell: pwsh
+
- name: Set NuGet Package Version Environment Variable from tag (only for tagged builds)
if: ${{ github.ref_type == 'tag' }}
run: echo "PKGVERSION=$('${{ github.ref_name }}'.replace('v',''))" >> $env:GITHUB_ENV
diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml
index 1a78a27..7745dbf 100644
--- a/.github/workflows/pr-build.yaml
+++ b/.github/workflows/pr-build.yaml
@@ -33,4 +33,12 @@ jobs:
- name: Run Tests in ReferenceCop.MSBuild.Tests
run: dotnet test --verbosity normal
- working-directory: src/ReferenceCop.MSBuild.Tests
\ No newline at end of file
+ working-directory: src/ReferenceCop.MSBuild.Tests
+
+ - name: Pack ReferenceCop for playground
+ run: dotnet pack -c Debug --nologo
+ working-directory: src/ReferenceCop.Package
+
+ - name: Run end-to-end tests
+ run: ./playground/Run-E2ETests.ps1
+ shell: pwsh
\ No newline at end of file
diff --git a/playground/Run-E2ETests.ps1 b/playground/Run-E2ETests.ps1
new file mode 100644
index 0000000..9394f6d
--- /dev/null
+++ b/playground/Run-E2ETests.ps1
@@ -0,0 +1,214 @@
+<#
+.SYNOPSIS
+ Automated end-to-end test runner for ReferenceCop playground scenarios.
+
+.DESCRIPTION
+ Builds various test configurations in the playground and validates that
+ ReferenceCop correctly detects violations and allows valid references.
+
+.PARAMETER PackFirst
+ If set, builds and packs the ReferenceCop package before running tests.
+#>
+param(
+ [switch]$PackFirst
+)
+
+$ErrorActionPreference = "Stop"
+$script:TestsPassed = 0
+$script:TestsFailed = 0
+$script:Failures = @()
+
+$RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path
+$PlaygroundRoot = "$PSScriptRoot/TestProject"
+
+function Write-TestHeader($name) {
+ Write-Host "`n========================================" -ForegroundColor Cyan
+ Write-Host " TEST: $name" -ForegroundColor Cyan
+ Write-Host "========================================" -ForegroundColor Cyan
+}
+
+function Assert-BuildFails($project, $testName, $expectedDiagnostic) {
+ Write-TestHeader $testName
+ $output = dotnet build "$PlaygroundRoot/$project" --no-restore 2>&1 | Out-String
+ $exitCode = $LASTEXITCODE
+
+ if ($exitCode -eq 0) {
+ Write-Host " FAIL: Build succeeded but was expected to fail" -ForegroundColor Red
+ Write-Host " Output: $output" -ForegroundColor Gray
+ $script:TestsFailed++
+ $script:Failures += $testName
+ return
+ }
+
+ if ($expectedDiagnostic -and ($output -notmatch $expectedDiagnostic)) {
+ Write-Host " FAIL: Build failed but diagnostic '$expectedDiagnostic' not found in output" -ForegroundColor Red
+ Write-Host " Output: $output" -ForegroundColor Gray
+ $script:TestsFailed++
+ $script:Failures += $testName
+ return
+ }
+
+ Write-Host " PASS: Build failed as expected with diagnostic '$expectedDiagnostic'" -ForegroundColor Green
+ $script:TestsPassed++
+}
+
+function Assert-BuildSucceeds($project, $testName) {
+ Write-TestHeader $testName
+ $output = dotnet build "$PlaygroundRoot/$project" --no-restore 2>&1 | Out-String
+ $exitCode = $LASTEXITCODE
+
+ if ($exitCode -ne 0) {
+ Write-Host " FAIL: Build failed but was expected to succeed" -ForegroundColor Red
+ Write-Host " Output: $output" -ForegroundColor Gray
+ $script:TestsFailed++
+ $script:Failures += $testName
+ return
+ }
+
+ Write-Host " PASS: Build succeeded as expected" -ForegroundColor Green
+ $script:TestsPassed++
+}
+
+# --- Setup ---
+
+if ($PackFirst) {
+ Write-Host "`nPacking ReferenceCop..." -ForegroundColor Yellow
+ Push-Location "$RepoRoot/src/ReferenceCop.Package"
+ dotnet pack -c Debug --nologo -v quiet
+ if ($LASTEXITCODE -ne 0) {
+ Write-Host "FATAL: Failed to pack ReferenceCop" -ForegroundColor Red
+ exit 1
+ }
+ Pop-Location
+}
+
+# Restore all playground projects
+Write-Host "`nRestoring playground projects..." -ForegroundColor Yellow
+dotnet restore "$PlaygroundRoot/SampleApp/SampleApp.csproj" --nologo -v quiet
+dotnet restore "$PlaygroundRoot/ValidApp/ValidApp.csproj" --nologo -v quiet
+
+# --- Test Scenarios ---
+
+# Scenario 1: AssemblyName rule - SampleLibrary references Newtonsoft.Json which is blocked
+Assert-BuildFails "SampleLibrary" `
+ "AssemblyName rule: Newtonsoft.Json blocked" `
+ "RC000[12]"
+
+# Scenario 2: ProjectPath rule - SampleApp referencing InternalLib should be blocked
+# We need to temporarily add a project reference for this test
+$sampleAppCsproj = "$PlaygroundRoot/SampleApp/SampleApp.csproj"
+$originalContent = Get-Content $sampleAppCsproj -Raw
+
+# Add InternalLib reference
+$modifiedContent = $originalContent -replace '(\s*)', @"
+
+
+
+
+
+
+
+"@
+# Fix: replace last properly
+$modifiedContent = $originalContent -replace '(\s*
+
+
+
+
+
+
+ Exe
+ net8.0
+ enable
+ enable
+ App
+
+
+
+
+
+
+
+
+"@
+
+# Save original, write modified, test, restore
+$originalSampleApp = Get-Content $sampleAppCsproj -Raw
+Set-Content $sampleAppCsproj -Value $testCsproj -NoNewline
+dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
+
+Assert-BuildFails "SampleApp" `
+ "ProjectPath rule: App cannot reference Internal" `
+ "RC000[12]"
+
+# Restore original
+Set-Content $sampleAppCsproj -Value $originalSampleApp -NoNewline
+dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
+
+# Scenario 3: ProjectTag rule - App referencing Tools project should warn
+$toolsTestCsproj = @"
+
+
+ Exe
+ net8.0
+ enable
+ enable
+ App
+
+
+
+
+
+
+
+
+"@
+
+Set-Content $sampleAppCsproj -Value $toolsTestCsproj -NoNewline
+dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
+
+# ProjectTag rule has Severity=Warning, so build may succeed but with warnings
+# We check for the diagnostic in output regardless of exit code
+Write-TestHeader "ProjectTag rule: App cannot reference Tools (warning)"
+$output = dotnet build "$PlaygroundRoot/SampleApp" --no-restore 2>&1 | Out-String
+if ($output -match "RC000[12]") {
+ Write-Host " PASS: ProjectTag violation diagnostic detected" -ForegroundColor Green
+ $script:TestsPassed++
+} else {
+ Write-Host " FAIL: Expected ProjectTag warning diagnostic not found" -ForegroundColor Red
+ Write-Host " Output: $output" -ForegroundColor Gray
+ $script:TestsFailed++
+ $script:Failures += "ProjectTag rule: App cannot reference Tools (warning)"
+}
+
+# Restore original
+Set-Content $sampleAppCsproj -Value $originalSampleApp -NoNewline
+dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
+
+# Scenario 4: Valid project builds successfully (no violations)
+Assert-BuildSucceeds "ValidApp" `
+ "Valid project: No violations, build succeeds"
+
+# --- Summary ---
+Write-Host "`n========================================" -ForegroundColor Cyan
+Write-Host " RESULTS" -ForegroundColor Cyan
+Write-Host "========================================" -ForegroundColor Cyan
+Write-Host " Passed: $script:TestsPassed" -ForegroundColor Green
+Write-Host " Failed: $script:TestsFailed" -ForegroundColor $(if ($script:TestsFailed -gt 0) { "Red" } else { "Green" })
+
+if ($script:TestsFailed -gt 0) {
+ Write-Host "`n Failed tests:" -ForegroundColor Red
+ foreach ($f in $script:Failures) {
+ Write-Host " - $f" -ForegroundColor Red
+ }
+ exit 1
+}
+
+Write-Host "`n All e2e tests passed!" -ForegroundColor Green
+exit 0
diff --git a/playground/TEST-SCENARIOS.md b/playground/TEST-SCENARIOS.md
new file mode 100644
index 0000000..547a080
--- /dev/null
+++ b/playground/TEST-SCENARIOS.md
@@ -0,0 +1,27 @@
+# E2E Test Scenarios
+
+This document describes the automated end-to-end test scenarios run by `Run-E2ETests.ps1`.
+
+## Scenarios
+
+| # | Rule Type | Scenario | Expected Result |
+|---|-----------|----------|-----------------|
+| 1 | AssemblyName | SampleLibrary references Newtonsoft.Json (blocked by `NoNewtonsoftJson` rule) | Build fails with RC0001/RC0002 |
+| 2 | ProjectPath | SampleApp references InternalLib (blocked by `AppCannotReferenceInternal` rule) | Build fails with RC0001/RC0002 |
+| 3 | ProjectTag | SampleApp (tag=App) references ToolsProject (tag=Tools), blocked by `AppCannotReferenceTools` rule | Warning diagnostic RC0001/RC0002 emitted |
+| 4 | Valid | ValidApp with no violations | Build succeeds cleanly |
+
+## Running Locally
+
+```powershell
+# Pack first, then run tests
+./playground/Run-E2ETests.ps1 -PackFirst
+
+# Or if already packed:
+./playground/Run-E2ETests.ps1
+```
+
+## CI Integration
+
+The tests run automatically in both PR and official build workflows after the unit tests pass.
+The workflow packs the package in Debug mode, then invokes the test runner.
diff --git a/playground/TestProject/InternalLib/InternalHelper.cs b/playground/TestProject/InternalLib/InternalHelper.cs
new file mode 100644
index 0000000..ae50d00
--- /dev/null
+++ b/playground/TestProject/InternalLib/InternalHelper.cs
@@ -0,0 +1,6 @@
+namespace InternalLib;
+
+public class InternalHelper
+{
+ public static string GetSecret() => "internal";
+}
diff --git a/playground/TestProject/InternalLib/InternalLib.csproj b/playground/TestProject/InternalLib/InternalLib.csproj
new file mode 100644
index 0000000..6f94e6a
--- /dev/null
+++ b/playground/TestProject/InternalLib/InternalLib.csproj
@@ -0,0 +1,10 @@
+
+
+
+ net8.0
+ enable
+ enable
+ Library
+
+
+
diff --git a/playground/TestProject/ToolsProject/Tool.cs b/playground/TestProject/ToolsProject/Tool.cs
new file mode 100644
index 0000000..ced6dcc
--- /dev/null
+++ b/playground/TestProject/ToolsProject/Tool.cs
@@ -0,0 +1,6 @@
+namespace ToolsProject;
+
+public class Tool
+{
+ public static void Run() { }
+}
diff --git a/playground/TestProject/ToolsProject/ToolsProject.csproj b/playground/TestProject/ToolsProject/ToolsProject.csproj
new file mode 100644
index 0000000..5ca2e66
--- /dev/null
+++ b/playground/TestProject/ToolsProject/ToolsProject.csproj
@@ -0,0 +1,10 @@
+
+
+
+ net8.0
+ enable
+ enable
+ Tools
+
+
+
diff --git a/playground/TestProject/ValidApp/Program.cs b/playground/TestProject/ValidApp/Program.cs
new file mode 100644
index 0000000..8012955
--- /dev/null
+++ b/playground/TestProject/ValidApp/Program.cs
@@ -0,0 +1 @@
+Console.WriteLine("Hello from ValidApp!");
diff --git a/playground/TestProject/ValidApp/ValidApp.csproj b/playground/TestProject/ValidApp/ValidApp.csproj
new file mode 100644
index 0000000..7d79ec2
--- /dev/null
+++ b/playground/TestProject/ValidApp/ValidApp.csproj
@@ -0,0 +1,15 @@
+
+
+
+ Exe
+ net8.0
+ enable
+ enable
+ App
+
+
+
+
+
+
+
diff --git a/playground/playground.slnx b/playground/playground.slnx
index c5ee38d..c0fecb1 100644
--- a/playground/playground.slnx
+++ b/playground/playground.slnx
@@ -7,4 +7,7 @@
+
+
+
From 2889b00c532f6417403c50a476711a921e4fd948 Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Mon, 25 May 2026 19:17:10 +0000
Subject: [PATCH 2/8] fix: use Release config for pack step to match build
output
---
.github/workflows/pr-build.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml
index 7745dbf..a2a7aa1 100644
--- a/.github/workflows/pr-build.yaml
+++ b/.github/workflows/pr-build.yaml
@@ -36,7 +36,7 @@ jobs:
working-directory: src/ReferenceCop.MSBuild.Tests
- name: Pack ReferenceCop for playground
- run: dotnet pack -c Debug --nologo
+ run: dotnet pack -c Release --nologo
working-directory: src/ReferenceCop.Package
- name: Run end-to-end tests
From c2d9eabced44adb88971d9e09612e2286cd4c027 Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Mon, 25 May 2026 19:46:41 +0000
Subject: [PATCH 3/8] fix: align playground nuget source with Release pack
configuration
---
playground/nuget.config | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/playground/nuget.config b/playground/nuget.config
index f45e370..3af9dc5 100644
--- a/playground/nuget.config
+++ b/playground/nuget.config
@@ -7,7 +7,7 @@
-
+
From 00a920fa139695cf93cdf8606d7a01d7b41d89cf Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Tue, 26 May 2026 20:18:50 +0000
Subject: [PATCH 4/8] fix: pass IsDevEnvironment=true to pack step for
playground e2e tests
---
.github/workflows/pr-build.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml
index a2a7aa1..d00b945 100644
--- a/.github/workflows/pr-build.yaml
+++ b/.github/workflows/pr-build.yaml
@@ -36,7 +36,7 @@ jobs:
working-directory: src/ReferenceCop.MSBuild.Tests
- name: Pack ReferenceCop for playground
- run: dotnet pack -c Release --nologo
+ run: dotnet pack -c Release --nologo -p:IsDevEnvironment=true
working-directory: src/ReferenceCop.Package
- name: Run end-to-end tests
From bd67d503a91d637a5236e92ed4cc2f3ecf741fc2 Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Wed, 27 May 2026 08:18:00 +0000
Subject: [PATCH 5/8] fix: narrow System.Web rule pattern to exact match for
net8.0 compat
In .NET 8, System.Web.HttpUtility is a legitimate shared framework
assembly. The wildcard pattern System.Web* incorrectly flags it in
ValidApp which has no actual System.Web dependency.
---
playground/TestProject/ReferenceCop.config | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/playground/TestProject/ReferenceCop.config b/playground/TestProject/ReferenceCop.config
index dce9c77..1bacda1 100644
--- a/playground/TestProject/ReferenceCop.config
+++ b/playground/TestProject/ReferenceCop.config
@@ -8,7 +8,7 @@
NoSystemWebReferences
System.Web assemblies should not be referenced
Error
- System.Web*
+ System.Web
From 62858f873fdf8f6d96999a344910679ac5688b6a Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Wed, 27 May 2026 11:17:15 +0000
Subject: [PATCH 6/8] fix: remove System.Web rule that conflicts with net8.0
framework refs
The System.Web assembly is part of the .NET 8 shared framework and
appears as a reference in all net8.0 projects. Removed the rule from
playground config since no test scenario exercises it. Changed
NoNewtonsoftJson to Error severity to ensure scenario 1 build fails.
---
playground/TestProject/ReferenceCop.config | 12 ++----------
1 file changed, 2 insertions(+), 10 deletions(-)
diff --git a/playground/TestProject/ReferenceCop.config b/playground/TestProject/ReferenceCop.config
index 1bacda1..3a903a2 100644
--- a/playground/TestProject/ReferenceCop.config
+++ b/playground/TestProject/ReferenceCop.config
@@ -3,15 +3,7 @@
true
true
-
-
- NoSystemWebReferences
- System.Web assemblies should not be referenced
- Error
- System.Web
-
-
-
+
AppCannotReferenceTools
Application projects should not reference tool projects
@@ -33,7 +25,7 @@
NoNewtonsoftJson
Use System.Text.Json instead of Newtonsoft.Json
- Warning
+ Error
Newtonsoft.Json*
From 4030715990ab20fc79b4dfef928bad48fc03c634 Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Wed, 27 May 2026 14:18:28 +0000
Subject: [PATCH 7/8] fix: use minimal Program.cs in e2e test scenarios 2 and 3
Scenarios 2 (ProjectPath) and 3 (ProjectTag) replace SampleApp.csproj
to test different reference rules. The original Program.cs has
'using SampleLibrary' which causes CS0246 when SampleLibrary isn't
referenced. Use a minimal Program.cs for these scenarios so the
ReferenceCop analyzer can run without compilation errors.
---
playground/Run-E2ETests.ps1 | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/playground/Run-E2ETests.ps1 b/playground/Run-E2ETests.ps1
index 9394f6d..acf6614 100644
--- a/playground/Run-E2ETests.ps1
+++ b/playground/Run-E2ETests.ps1
@@ -138,17 +138,23 @@ $testCsproj = @"
"@
-# Save original, write modified, test, restore
+# Save originals, write modified project and minimal program
$originalSampleApp = Get-Content $sampleAppCsproj -Raw
+$sampleAppProgram = "$PlaygroundRoot/SampleApp/Program.cs"
+$originalProgram = Get-Content $sampleAppProgram -Raw
+$minimalProgram = "namespace SampleApp;`npublic class Program { public static void Main() { } }"
+
Set-Content $sampleAppCsproj -Value $testCsproj -NoNewline
+Set-Content $sampleAppProgram -Value $minimalProgram -NoNewline
dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
Assert-BuildFails "SampleApp" `
"ProjectPath rule: App cannot reference Internal" `
"RC000[12]"
-# Restore original
+# Restore originals
Set-Content $sampleAppCsproj -Value $originalSampleApp -NoNewline
+Set-Content $sampleAppProgram -Value $originalProgram -NoNewline
dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
# Scenario 3: ProjectTag rule - App referencing Tools project should warn
@@ -171,6 +177,7 @@ $toolsTestCsproj = @"
"@
Set-Content $sampleAppCsproj -Value $toolsTestCsproj -NoNewline
+Set-Content $sampleAppProgram -Value $minimalProgram -NoNewline
dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
# ProjectTag rule has Severity=Warning, so build may succeed but with warnings
@@ -187,8 +194,9 @@ if ($output -match "RC000[12]") {
$script:Failures += "ProjectTag rule: App cannot reference Tools (warning)"
}
-# Restore original
+# Restore originals
Set-Content $sampleAppCsproj -Value $originalSampleApp -NoNewline
+Set-Content $sampleAppProgram -Value $originalProgram -NoNewline
dotnet restore "$sampleAppCsproj" --nologo -v quiet 2>$null
# Scenario 4: Valid project builds successfully (no violations)
From ae31e2d9a7a432dfe9896dd51bfab1512efcb95b Mon Sep 17 00:00:00 2001
From: mfogliatto <2962955+mfogliatto@users.noreply.github.com>
Date: Wed, 27 May 2026 20:16:19 +0000
Subject: [PATCH 8/8] fix: correct ProjectPath rule ToPath to match InternalLib
directory
The ToPath pattern was '**/Internal/**' but the actual test project
directory is 'InternalLib', not 'Internal'.
---
playground/TestProject/ReferenceCop.config | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/playground/TestProject/ReferenceCop.config b/playground/TestProject/ReferenceCop.config
index 3a903a2..b196501 100644
--- a/playground/TestProject/ReferenceCop.config
+++ b/playground/TestProject/ReferenceCop.config
@@ -18,7 +18,7 @@
Applications should not reference internal implementation libraries
Error
**/SampleApp/**
- **/Internal/**
+ **/InternalLib/**