diff --git a/.github/agents/foundry-repo-auditor.agent.md b/.github/agents/foundry-repo-auditor.agent.md new file mode 100644 index 000000000..eaa48fa64 --- /dev/null +++ b/.github/agents/foundry-repo-auditor.agent.md @@ -0,0 +1,51 @@ +--- +name: Foundry Repo Auditor +description: Scan this repository for folders that can become Azure AI Foundry agents, verify deployment state when project metadata is available, and prepare deduplicated GitHub issues for follow-up. +target: github-copilot +tools: ["read", "search", "edit", "execute", "github/*", "Azure/*"] +mcp-servers: + Azure: + type: local + command: npx + args: ["-y", "@azure/mcp@latest", "server", "start"] + tools: ["*"] +--- + +# Foundry Repo Auditor + +You are the repository specialist for identifying Azure AI Foundry agent candidates in this repo. + +## When to use this agent + +Use this agent when asked to: +- scan the repository for folders that can become Foundry agents +- determine whether a sample appears to already be deployed to Foundry +- prepare or create one GitHub issue per candidate or per problem found +- re-audit the repo after sample changes + +## Workflow + +1. Load and follow the repo-local `foundry-repo-audit` skill for the detailed audit procedure. +2. Respect `.github/copilot-instructions.md` and `.github/CODEOWNERS` boundaries. Do not rename, move, or otherwise modify docs-owned sample files unless the user explicitly asks and the requested change is allowed. +3. During deployment verification, use the Azure MCP Server Foundry tools when they are available in this agent's MCP configuration. +4. Prefer a report-first workflow. If the user explicitly asks to create issues, search for duplicates and then create GitHub issues. +5. Keep issue creation idempotent by using the dedupe token `foundry-agent-audit::` in every audit issue title or body. +6. For every candidate, capture classification, evidence, resolved metadata, and deployment state (`verified`, `not deployed`, or `unknown`). +7. If endpoint or agent-name metadata is missing, or Azure MCP is unavailable, do not guess; mark deployment state as `unknown` and explain what is missing. + +## Required output + +Produce a compact summary for every candidate that includes: +- candidate path +- readiness classification +- evidence +- resolved endpoint and agent name, if available +- deployment state +- issue status (`planned`, `created`, or `skipped as duplicate`) + +## Guardrails + +- Never create duplicate issues for the same dedupe token. +- Never claim deployment verification unless it came from Azure MCP Foundry tools or clearly stored repo metadata. +- Do not open issues for folders that clearly do not represent agents or convertible samples. +- Keep issue bodies actionable and specific to the path being audited. diff --git a/.github/copilot/azure-foundry-mcp.json b/.github/copilot/azure-foundry-mcp.json new file mode 100644 index 000000000..8acc19d0e --- /dev/null +++ b/.github/copilot/azure-foundry-mcp.json @@ -0,0 +1,10 @@ +{ + "mcpServers": { + "Azure": { + "type": "local", + "command": "npx", + "args": ["-y", "@azure/mcp@latest", "server", "start"], + "tools": ["*"] + } + } +} diff --git a/.github/scripts/foundry-repo-audit-prompt.md b/.github/scripts/foundry-repo-audit-prompt.md new file mode 100644 index 000000000..6318a4af7 --- /dev/null +++ b/.github/scripts/foundry-repo-audit-prompt.md @@ -0,0 +1,20 @@ +Audit this repository with the `foundry-repo-audit` skill. + +Focus on candidate folders that can become Azure AI Foundry agents and classify them as either: +- `ready-hosted-agent` +- `convertible-sample` + +For each candidate: +- record the relative path +- record concrete evidence used for classification +- resolve any available `AZURE_AI_PROJECT_ENDPOINT`, `AZURE_AI_AGENT_NAME`, `AGENT_NAME`, or `agentName` metadata and note the source +- verify deployment status when both project endpoint and agent name are available +- otherwise mark deployment state as `unknown` and list the missing metadata + +Produce a concise final report that summarizes: +- candidates reviewed +- classifications +- deployment states +- issue actions taken + +If issue creation is enabled by the workflow, use the dedupe token format `foundry-agent-audit::`. diff --git a/.github/skills/foundry-repo-audit/SKILL.md b/.github/skills/foundry-repo-audit/SKILL.md new file mode 100644 index 000000000..be4c2359e --- /dev/null +++ b/.github/skills/foundry-repo-audit/SKILL.md @@ -0,0 +1,97 @@ +--- +name: foundry-repo-audit +description: Repository-specific workflow for discovering Azure AI Foundry agent candidates, resolving local deployment metadata, verifying deployment with Azure MCP Foundry tools, and preparing deduplicated GitHub issues. +argument-hint: "[optional path scope or sample family]" +user-invocable: true +--- + +# Foundry Repo Audit + +Use this skill when working in `xoeest-foundry-samples` and the task is to find folders that can become Azure AI Foundry agents, determine whether those folders already map to deployed Foundry agents, or open audit issues for follow-up. + +## Repository-specific guidance + +- Hosted-agent candidates in this repo are primarily signaled by `agent.yaml`. +- Convertible samples often have a `Dockerfile`, a language manifest (`requirements.txt`, `pyproject.toml`, `package.json`, `*.csproj`, `go.mod`, or `pom.xml`), an entrypoint, and a `README.md`. +- Quickstart samples with `sample.yaml` are useful secondary candidates, but they should normally be classified as `convertible` instead of `ready` unless they also have hosted-agent signals. +- Existing GitHub issues can fan out to Azure DevOps through `.github/workflows/ado-automation.yml`, so GitHub issues are the preferred write target. +- Docs-owned files listed in `.github/CODEOWNERS` must not be renamed or moved. + +## Candidate discovery workflow + +1. Search for `**/agent.yaml` and classify each parent folder as a `ready-hosted-agent` candidate. +2. Search for `**/sample.yaml` and classify each parent folder as a `convertible-sample` candidate unless stronger hosted-agent evidence is present. +3. For folders without `agent.yaml`, look for the following combination before treating them as convertible: + - `Dockerfile` + - one language manifest + - an obvious entrypoint such as `main.py`, `Program.cs`, `index.ts`, `app.py`, or another runnable source file + - a `README.md` +4. Exclude `.github`, `.infra`, `infrastructure`, `migration`, test folders, and docs-only folders unless the user explicitly asks for a broader audit. +5. Record concrete evidence for every candidate instead of using generic labels like "looks deployable." + +## Metadata resolution workflow + +For each candidate, resolve deployment metadata in this order: + +1. Search the candidate folder and its nearby config files for: + - `AZURE_AI_PROJECT_ENDPOINT` + - `AZURE_AI_AGENT_NAME` + - `AGENT_NAME` + - `agentName` +2. Prefer local runtime/config sources over documentation examples: + - `.env*` + - `azure.yaml` + - `launchSettings.json` + - PowerShell or shell deployment scripts +3. Use `agent.yaml` as a fallback source: + - top-level `name` + - `template.name` + - `environment_variables` +4. Use README snippets only as low-confidence hints when no stronger config exists. +5. Track both the resolved value and its source. If a value is not present, leave it unset instead of inventing one. + +## Foundry deployment verification + +When both a project endpoint and agent name are available: + +1. Use the Azure MCP Server Foundry tools configured for this repository or agent. +2. Prefer Foundry agent listing and lookup operations to check whether the agent exists for the resolved project endpoint and agent name. +3. Classify the result as: + - `verified` when the agent lookup succeeds + - `not deployed` when the lookup definitively reports the agent does not exist + - `unknown` when credentials, connectivity, or incomplete metadata prevent verification + +When either the project endpoint or agent name is missing, or when Azure MCP is not configured, set deployment state to `unknown` and list the missing fields. + +If you are running locally in a richer environment that also provides an external Foundry plugin skill, you may use it as an enhancement, but the cloud path must not depend on that plugin being present. + +## GitHub issue workflow + +Before creating an issue: + +1. Search existing issues for the dedupe token `foundry-agent-audit::`. +2. Reuse `./issue-template.md` for the issue structure. +3. Default to one issue per candidate path unless the user explicitly asks for one issue per problem type. +4. Recommended labels: + - `foundry-agent-audit` + - readiness label such as `ready-hosted-agent` or `convertible-sample` + - an optional language label if it can be inferred safely + +## Expected audit output + +For each candidate, report: +- relative path +- classification +- evidence +- resolved project endpoint and agent name with sources +- deployment state +- issue action taken (`planned`, `created`, or `duplicate`) + +## Useful repo references + +- `.github/workflows/ado-automation.yml` +- `.github/copilot/azure-foundry-mcp.json` +- `.github/workflows/copilot-setup-steps.yml` +- `samples/python/hosted-agents/agent-framework/echo-agent/agent.yaml` +- `samples/csharp/FoundryA365/azure.yaml` +- `.github/CODEOWNERS` diff --git a/.github/skills/foundry-repo-audit/issue-template.md b/.github/skills/foundry-repo-audit/issue-template.md new file mode 100644 index 000000000..0664dddbb --- /dev/null +++ b/.github/skills/foundry-repo-audit/issue-template.md @@ -0,0 +1,29 @@ +# [Foundry agent audit] {{ relative_path }} + +Dedupe token: `foundry-agent-audit::{{ relative_path }}` + +## Audit summary + +- Classification: `{{ classification }}` +- Deployment state: `{{ deployment_state }}` +- Agent name: `{{ agent_name_or_unknown }}` +- Project endpoint: `{{ project_endpoint_or_unknown }}` + +## Evidence + +- `{{ evidence_1 }}` +- `{{ evidence_2 }}` + +## Missing data or blockers + +- `{{ blocker_1 }}` + +## Recommended next actions + +- [ ] `{{ next_action_1 }}` +- [ ] `{{ next_action_2 }}` + +## Notes + +- Metadata source(s): `{{ metadata_sources }}` +- Audit path: `{{ relative_path }}` diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 000000000..bc0ac84b3 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,22 @@ +# This workflow configures Azure authentication for GitHub Copilot coding agent MCP sessions. +on: + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + copilot-setup-steps: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + environment: copilot + steps: + - name: Azure login + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + allow-no-subscriptions: true diff --git a/.github/workflows/foundry-repo-audit.yml b/.github/workflows/foundry-repo-audit.yml new file mode 100644 index 000000000..07443bda2 --- /dev/null +++ b/.github/workflows/foundry-repo-audit.yml @@ -0,0 +1,79 @@ +name: Foundry Repo Audit + +on: + push: + branches: + - main + workflow_dispatch: + inputs: + create_issues: + description: Create deduplicated GitHub issues instead of running in report-only mode + required: false + type: boolean + default: false + +permissions: + contents: read + issues: write + +concurrency: + group: foundry-repo-audit-${{ github.ref }} + cancel-in-progress: true + +jobs: + audit: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install GitHub Copilot CLI + run: npm install -g @github/copilot + + - name: Validate Copilot token + env: + COPILOT_PAT: ${{ secrets.COPILOT_PAT }} + run: | + if [ -z "$COPILOT_PAT" ]; then + echo "::error::Missing repository secret COPILOT_PAT." + echo "::error::Create a fine-grained PAT with Copilot Requests permission and repository access to read contents and write issues." + exit 1 + fi + + - name: Run Foundry repo auditor + env: + GH_TOKEN: ${{ secrets.COPILOT_PAT }} + GITHUB_TOKEN: ${{ secrets.COPILOT_PAT }} + CREATE_ISSUES: ${{ github.event_name == 'workflow_dispatch' && inputs.create_issues || 'false' }} + run: | + BASE_PROMPT="$(cat .github/scripts/foundry-repo-audit-prompt.md)" + + if [ "$CREATE_ISSUES" = "true" ]; then + MODE_PROMPT="Create deduplicated GitHub issues when needed. Before creating any issue, search for the dedupe token and skip duplicates." + else + MODE_PROMPT="Run in report-only mode. Do not create or modify GitHub issues." + fi + + FULL_PROMPT="${BASE_PROMPT}"$'\n\n'"${MODE_PROMPT}" + + copilot \ + --allow-all \ + --enable-all-github-mcp-tools \ + --no-ask-user \ + --agent foundry-repo-auditor \ + --prompt "$FULL_PROMPT" \ + --silent \ + --share "$RUNNER_TEMP/foundry-repo-audit.md" + + - name: Upload audit report + uses: actions/upload-artifact@v4 + with: + name: foundry-repo-audit-report + path: ${{ runner.temp }}/foundry-repo-audit.md diff --git a/README.md b/README.md index ab07f0b71..c3e583451 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,62 @@ This repository is entirely open source, guidance on how to contribute and links Use the samples in this repository to try out Azure AI Foundry scenarios on your local machine! +## Foundry repo audit automation + +This repo includes a GitHub Copilot custom agent and a GitHub Actions workflow for auditing folders that can become Azure AI Foundry agents. + +### What was added + +- Custom agent: `.github/agents/foundry-repo-auditor.agent.md` +- Repo skill: `.github/skills/foundry-repo-audit/SKILL.md` +- Workflow: `.github/workflows/foundry-repo-audit.yml` +- Azure MCP reference config: `.github/copilot/azure-foundry-mcp.json` +- Copilot coding agent setup workflow: `.github/workflows/copilot-setup-steps.yml` + +### Default behavior + +- The workflow runs automatically on every push to `main`. +- The workflow can also be started manually from the **Actions** tab by running **Foundry Repo Audit**. +- Automatic runs are report-only and upload an audit artifact instead of creating issues. + +### Required secret + +Create a repository secret named `COPILOT_PAT` before using the workflow. + +Recommended token setup: +- token type: fine-grained personal access token +- GitHub permission: **Copilot Requests** +- repository access: this repository +- repository permissions: enough access to read contents and create issues + +To add the secret on GitHub.com: +1. Open the repository. +2. Go to **Settings** > **Secrets and variables** > **Actions**. +3. Select **New repository secret**. +4. Name it `COPILOT_PAT`. +5. Paste the token value and save. + +### Azure MCP / Foundry setup for GitHub coding agent + +To let the GitHub website coding agent or custom agent verify Foundry deployments in the cloud, finish the Azure MCP setup in GitHub: + +1. Open **Settings** > **Copilot** > **Coding agent**. +2. In **MCP configuration**, paste the JSON from `.github/copilot/azure-foundry-mcp.json`. +3. Open **Settings** > **Environments** and create an environment named `copilot`. +4. Add environment secrets named `AZURE_CLIENT_ID` and `AZURE_TENANT_ID`. +5. Run the **copilot-setup-steps** workflow once from the **Actions** tab to validate Azure login for Copilot coding agent sessions. + +This repo now includes agent-level Azure MCP configuration in `.github/agents/foundry-repo-auditor.agent.md`, but the GitHub website experience still depends on the repository Copilot environment being configured correctly. + +### How to run it from the GitHub website + +1. Open the repository on GitHub. +2. Select the **Actions** tab. +3. Choose **Foundry Repo Audit** from the workflow list. +4. Select **Run workflow**. +5. If you want the run to create GitHub issues, enable the `create_issues` option. +6. Start the workflow and then open the run summary to review logs and download the `foundry-repo-audit-report` artifact. + ## Contributing Found a bug or have a suggestion? [Open an issue](https://github.com/microsoft-foundry/foundry-samples/issues/new) — we welcome feedback from everyone! diff --git a/tests/unit/test_copilot_customizations.py b/tests/unit/test_copilot_customizations.py new file mode 100644 index 000000000..4343d26e5 --- /dev/null +++ b/tests/unit/test_copilot_customizations.py @@ -0,0 +1,102 @@ +import json +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +AGENT_FILE = REPO_ROOT / ".github" / "agents" / "foundry-repo-auditor.agent.md" +SKILL_FILE = REPO_ROOT / ".github" / "skills" / "foundry-repo-audit" / "SKILL.md" +ISSUE_TEMPLATE = REPO_ROOT / ".github" / "skills" / "foundry-repo-audit" / "issue-template.md" +WORKFLOW_FILE = REPO_ROOT / ".github" / "workflows" / "foundry-repo-audit.yml" +PROMPT_FILE = REPO_ROOT / ".github" / "scripts" / "foundry-repo-audit-prompt.md" +SETUP_WORKFLOW_FILE = REPO_ROOT / ".github" / "workflows" / "copilot-setup-steps.yml" +MCP_CONFIG_FILE = REPO_ROOT / ".github" / "copilot" / "azure-foundry-mcp.json" + + +def parse_frontmatter(path: Path) -> tuple[dict[str, str], str]: + content = path.read_text(encoding="utf-8") + assert content.startswith("---\n"), f"{path} is missing YAML frontmatter" + + _, frontmatter, body = content.split("---\n", 2) + metadata: dict[str, str] = {} + + for line in frontmatter.splitlines(): + if not line.strip(): + continue + if line[:1].isspace(): + continue + key, value = line.split(":", 1) + metadata[key.strip()] = value.strip().strip('"').strip("'") + + return metadata, body + + +def test_custom_agent_profile_has_required_metadata_and_instructions() -> None: + metadata, body = parse_frontmatter(AGENT_FILE) + + assert metadata["name"] == "Foundry Repo Auditor" + assert metadata["target"] == "github-copilot" + assert "Azure/*" in metadata["tools"] + assert "mcp-servers" in metadata + assert "Azure AI Foundry agents" in metadata["description"] + assert "foundry-repo-audit" in body + assert "Azure MCP Server Foundry tools" in body + assert "foundry-agent-audit::" in body + + +def test_repo_skill_matches_directory_and_mentions_core_workflow_steps() -> None: + metadata, body = parse_frontmatter(SKILL_FILE) + + assert metadata["name"] == "foundry-repo-audit" + assert metadata["argument-hint"] == "[optional path scope or sample family]" + assert metadata["user-invocable"] == "true" + assert "`agent.yaml`" in body + assert "`sample.yaml`" in body + assert "`AZURE_AI_PROJECT_ENDPOINT`" in body + assert "Azure MCP Server Foundry tools" in body + assert "`./issue-template.md`" in body + + +def test_issue_template_contains_dedupe_token_and_placeholders() -> None: + content = ISSUE_TEMPLATE.read_text(encoding="utf-8") + + assert "foundry-agent-audit::{{ relative_path }}" in content + assert "{{ classification }}" in content + assert "{{ deployment_state }}" in content + assert "{{ next_action_1 }}" in content + + +def test_workflow_runs_on_main_and_uses_custom_agent() -> None: + content = WORKFLOW_FILE.read_text(encoding="utf-8") + + assert "push:" in content + assert "branches:" in content + assert "- main" in content + assert "workflow_dispatch:" in content + assert "COPILOT_PAT" in content + assert "--agent foundry-repo-auditor" in content + assert ".github/scripts/foundry-repo-audit-prompt.md" in content + assert "actions/upload-artifact@v4" in content + + +def test_prompt_file_mentions_skill_and_dedupe_token() -> None: + content = PROMPT_FILE.read_text(encoding="utf-8") + + assert "`foundry-repo-audit` skill" in content + assert "`AZURE_AI_PROJECT_ENDPOINT`" in content + assert "`AZURE_AI_AGENT_NAME`" in content + assert "foundry-agent-audit::" in content + + +def test_setup_workflow_and_mcp_config_exist_for_coding_agent() -> None: + workflow = SETUP_WORKFLOW_FILE.read_text(encoding="utf-8") + config = json.loads(MCP_CONFIG_FILE.read_text(encoding="utf-8")) + + assert "workflow_dispatch:" in workflow + assert "environment: copilot" in workflow + assert "azure/login@" in workflow + assert "AZURE_CLIENT_ID" in workflow + assert "AZURE_TENANT_ID" in workflow + + assert config["mcpServers"]["Azure"]["command"] == "npx" + assert config["mcpServers"]["Azure"]["args"] == ["-y", "@azure/mcp@latest", "server", "start"] + assert config["mcpServers"]["Azure"]["tools"] == ["*"]