diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 922bdf6..f49ba62 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -908,10 +908,14 @@ jobs: done < "$DIGEST_FILE" record-connector-registry: - # require binaries to succeed; windows and docker may be skipped (msi=false, docker=false && lambda=false) + # Legacy dist recording: manifest + S3 upload. + # Require binaries to succeed; windows and docker may be skipped based on inputs. + # Each optional job must succeed if it ran — a failure means incomplete release artifacts. # see: https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution - if: ${{ !cancelled() && needs.goreleaser-binaries.result == 'success' && (needs.goreleaser-windows.result == 'success' || needs.goreleaser-windows.result == 'skipped') }} + if: ${{ !cancelled() && needs.goreleaser-binaries.result == 'success' && (needs.goreleaser-windows.result == 'success' || needs.goreleaser-windows.result == 'skipped') && (needs.goreleaser-docker.result == 'success' || needs.goreleaser-docker.result == 'skipped') }} needs: [determine-workflows-ref, goreleaser-binaries, goreleaser-windows, goreleaser-docker] + outputs: + merged_manifest: ${{ steps.export-manifest.outputs.merged_manifest }} permissions: id-token: write contents: read @@ -1061,6 +1065,14 @@ jobs: --output-signature "manifest.json.sig" \ --output-certificate "manifest.json.cert" + - name: Export final manifest for registry API + id: export-manifest + working-directory: _workflows/_output + run: | + # Output the final merged+signed manifest as a job output + # so record-registry-api can use the exact same manifest. + echo "merged_manifest=$(cat manifest.json | jq -c .)" >> "$GITHUB_OUTPUT" + - name: Upload checksums to S3 working-directory: _workflows/_output env: @@ -1179,7 +1191,10 @@ jobs: record-lambda-registry: if: inputs.lambda == true - # lambda releases are dependent on assets produced in the binaries and docker goreleaser jobs + # Legacy per-connector Lambda invocation — records to connectorreleases DynamoDB. + # Only needs binaries + docker (container images). Does not gate on windows/msi since + # the Lambda pipeline only cares about container images. Will be removed after cutover + # to the registry API (record-connector-registry replaces this path). needs: [goreleaser-binaries, goreleaser-docker] permissions: id-token: write @@ -1244,6 +1259,114 @@ jobs: rm -f "$TMPFILE" + # ================================================================ + # Registry API: record release after all legacy recording completes. + # Depends on both dist (record-connector-registry) and lambda (record-lambda-registry) + # so the recording has the full picture: assets, images, config_schema, capabilities. + # continue-on-error on the recording step so failures don't block the release. + # Will become the sole recording path after cutover. + # ================================================================ + record-registry-api: + if: ${{ !cancelled() && needs.record-connector-registry.result == 'success' && (needs.record-lambda-registry.result == 'success' || needs.record-lambda-registry.result == 'skipped') }} + needs: [determine-workflows-ref, record-connector-registry, record-lambda-registry] + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + steps: + - name: Checkout connector workflows + uses: actions/checkout@v5 + with: + path: _workflows + repository: ConductorOne/github-workflows + ref: ${{ needs.determine-workflows-ref.outputs.ref }} + + - name: Set up Go for workflows + uses: actions/setup-go@v6 + with: + go-version-file: "_workflows/go.mod" + + - name: Checkout connector repo + uses: actions/checkout@v5 + with: + path: _connector + + - name: Read connector documentation + id: read-docs + run: | + if [ -f "_connector/docs/connector.mdx" ]; then + echo "Found docs/connector.mdx" + echo "has_docs=true" >> "$GITHUB_OUTPUT" + else + echo "No docs/connector.mdx found" + echo "has_docs=false" >> "$GITHUB_OUTPUT" + fi + + - name: Get GitHub OIDC token for registry API + id: registry-oidc + uses: actions/github-script@v7 + with: + script: | + const token = await core.getIDToken('connector-registry') + core.setSecret(token) + core.setOutput('token', token) + + - name: Fetch release notes for changelog + if: steps.registry-oidc.outcome == 'success' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api "repos/${{ github.repository }}/releases/tags/${{ inputs.tag }}" --jq .body > /tmp/changelog.md || true + + - name: Write merged manifest from dist recording job + working-directory: _workflows + env: + MERGED_MANIFEST: ${{ needs.record-connector-registry.outputs.merged_manifest }} + run: | + mkdir -p _output + echo "$MERGED_MANIFEST" | jq . > _output/manifest.json + + - name: Record release via registry API + if: steps.registry-oidc.outcome == 'success' + working-directory: _workflows + env: + REGISTRY_API_TOKEN: ${{ steps.registry-oidc.outputs.token }} + run: | + DOCS_FLAG="" + if [ "${{ steps.read-docs.outputs.has_docs }}" = "true" ]; then + DOCS_FLAG="-docs ../_connector/docs/connector.mdx" + fi + + CHANGELOG_FLAG="" + if [ -s /tmp/changelog.md ]; then + CHANGELOG_FLAG="-changelog /tmp/changelog.md" + fi + + CONFIG_SCHEMA_FLAG="" + if [ -f "../_connector/config_schema.json" ]; then + CONFIG_SCHEMA_FLAG="-config-schema ../_connector/config_schema.json" + fi + + CAPABILITIES_FLAG="" + if [ -f "../_connector/baton_capabilities.json" ]; then + CAPABILITIES_FLAG="-capabilities ../_connector/baton_capabilities.json" + fi + + go run ./cmd/record-release \ + -manifest _output/manifest.json \ + -org "${{ github.event.repository.owner.login }}" \ + -name "${{ github.event.repository.name }}" \ + -version "${{ inputs.tag }}" \ + -repository-url "https://github.com/${{ github.repository }}" \ + -commit-sha "${{ github.sha }}" \ + -workflow-run-id "${{ github.run_id }}" \ + -registry-url "https://dist.conductorone.com" \ + $DOCS_FLAG \ + $CHANGELOG_FLAG \ + $CONFIG_SCHEMA_FLAG \ + $CAPABILITIES_FLAG + verify-release: # Verify release artifacts and attestations after publishing # This job is not blocking - failures trigger Datadog notification but don't fail the release @@ -1290,6 +1413,7 @@ jobs: goreleaser-docker, record-connector-registry, record-lambda-registry, + record-registry-api, verify-release, ] if: failure() diff --git a/cmd/record-release/main.go b/cmd/record-release/main.go new file mode 100644 index 0000000..48e4601 --- /dev/null +++ b/cmd/record-release/main.go @@ -0,0 +1,311 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "google.golang.org/protobuf/encoding/protojson" + + pb "github.com/ConductorOne/github-workflows/pb/artifacts/v1" +) + +// RecordReleaseRequest is the JSON body sent to the registry API. +type RecordReleaseRequest struct { + Org string `json:"org"` + Name string `json:"name"` + Version string `json:"version"` + RepositoryURL string `json:"repositoryUrl"` + CommitSha string `json:"commitSha"` + WorkflowRunID string `json:"workflowRunId"` + Documentation string `json:"documentation,omitempty"` + Changelog string `json:"changelog,omitempty"` + ConfigSchema string `json:"configSchema,omitempty"` + Capabilities string `json:"capabilities,omitempty"` + SignatureURL string `json:"signatureUrl,omitempty"` + CertificateURL string `json:"certificateUrl,omitempty"` + Assets map[string]*ReleaseAsset `json:"assets,omitempty"` + Images map[string]*ReleaseImage `json:"images,omitempty"` +} + +// ReleaseAsset is the transformed asset for the registry API. +type ReleaseAsset struct { + Platform string `json:"platform"` + Filename string `json:"filename"` + MediaType string `json:"mediaType"` + SizeBytes int64 `json:"sizeBytes"` + Sha256 string `json:"sha256"` + DownloadURL string `json:"downloadUrl"` + SignatureURL string `json:"signatureUrl,omitempty"` + CertificateURL string `json:"certificateUrl,omitempty"` + SbomURL string `json:"sbomUrl,omitempty"` +} + +// ReleaseImage is the transformed image for the registry API. +type ReleaseImage struct { + Ref string `json:"ref"` + Digest string `json:"digest"` + Platform string `json:"platform"` +} + +// authTransport adds a Bearer token to every outgoing request. +type authTransport struct { + token string + base http.RoundTripper +} + +func (t *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.Header.Set("Authorization", "Bearer "+t.token) + req.Header.Set("Content-Type", "application/json") + return t.base.RoundTrip(req) +} + +func main() { + var ( + manifestPath string + docsPath string + org string + name string + version string + repositoryURL string + commitSha string + workflowRunID string + registryURL string + changelogPath string + configSchemaPath string + capabilitiesPath string + token string + ) + + flag.StringVar(&manifestPath, "manifest", "", "Path to merged manifest.json file (required)") + flag.StringVar(&docsPath, "docs", "", "Path to docs/connector.mdx file (optional)") + flag.StringVar(&org, "org", "", "GitHub organization (required)") + flag.StringVar(&name, "name", "", "Repository/connector name (required)") + flag.StringVar(&version, "version", "", "Release version tag (required)") + flag.StringVar(&repositoryURL, "repository-url", "", "Full repository URL (required)") + flag.StringVar(&commitSha, "commit-sha", "", "Git commit SHA (required)") + flag.StringVar(&workflowRunID, "workflow-run-id", "", "GitHub Actions workflow run ID (required)") + flag.StringVar(®istryURL, "registry-url", "", "Registry API base URL (required)") + flag.StringVar(&changelogPath, "changelog", "", "Path to a file containing release notes (optional)") + flag.StringVar(&configSchemaPath, "config-schema", "", "Path to config_schema.json file (optional)") + flag.StringVar(&capabilitiesPath, "capabilities", "", "Path to baton_capabilities.json file (optional)") + flag.StringVar(&token, "token", "", "Bearer token (or set REGISTRY_API_TOKEN env var)") + flag.Parse() + + // Validate required flags + var missing []string + if manifestPath == "" { + missing = append(missing, "-manifest") + } + if org == "" { + missing = append(missing, "-org") + } + if name == "" { + missing = append(missing, "-name") + } + if version == "" { + missing = append(missing, "-version") + } + if repositoryURL == "" { + missing = append(missing, "-repository-url") + } + if commitSha == "" { + missing = append(missing, "-commit-sha") + } + if workflowRunID == "" { + missing = append(missing, "-workflow-run-id") + } + if registryURL == "" { + missing = append(missing, "-registry-url") + } + if len(missing) > 0 { + fmt.Fprintf(os.Stderr, "record-release: error: missing required flags: %s\n", strings.Join(missing, ", ")) + flag.Usage() + os.Exit(1) + } + + // Resolve token: flag > env var + if token == "" { + token = os.Getenv("REGISTRY_API_TOKEN") + } + if token == "" { + fmt.Fprintf(os.Stderr, "record-release: error: bearer token required (use -token flag or REGISTRY_API_TOKEN env var)\n") + os.Exit(1) + } + + // Read and parse manifest using protojson (same pattern as merge-manifests) + manifestBytes, err := os.ReadFile(manifestPath) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: error: reading manifest: %v\n", err) + os.Exit(1) + } + + manifest := &pb.Manifest{} + unmarshalOpts := protojson.UnmarshalOptions{ + DiscardUnknown: true, + } + if err := unmarshalOpts.Unmarshal(manifestBytes, manifest); err != nil { + fmt.Fprintf(os.Stderr, "record-release: error: parsing manifest: %v\n", err) + os.Exit(1) + } + + // Read optional documentation + var documentation string + if docsPath != "" { + docsBytes, err := os.ReadFile(docsPath) + if err != nil { + // Not fatal -- docs are optional + fmt.Fprintf(os.Stderr, "record-release: warning: could not read docs file: %v\n", err) + } else { + documentation = string(docsBytes) + } + } + + // Read optional changelog / release notes + var changelog string + if changelogPath != "" { + changelogBytes, err := os.ReadFile(changelogPath) + if err != nil { + // Not fatal -- changelog is optional + fmt.Fprintf(os.Stderr, "record-release: warning: could not read changelog file: %v\n", err) + } else { + changelog = string(changelogBytes) + } + } + + // Read optional config_schema.json (committed to connector repo by CI) + var configSchema string + if configSchemaPath != "" { + data, err := os.ReadFile(configSchemaPath) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: warning: could not read config-schema file: %v\n", err) + } else { + configSchema = string(data) + } + } + + // Read optional baton_capabilities.json (committed to connector repo by CI) + var capabilities string + if capabilitiesPath != "" { + data, err := os.ReadFile(capabilitiesPath) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: warning: could not read capabilities file: %v\n", err) + } else { + capabilities = string(data) + } + } + + // Transform manifest assets: href->downloadUrl, signatureHref->signatureUrl, etc. + assets := make(map[string]*ReleaseAsset) + for platform, asset := range manifest.GetAssets() { + assets[platform] = &ReleaseAsset{ + Platform: platform, + Filename: asset.GetFilename(), + MediaType: asset.GetMediaType(), + SizeBytes: asset.GetSizeBytes(), + Sha256: asset.GetSha256(), + DownloadURL: asset.GetHref(), + SignatureURL: asset.GetSignatureHref(), + CertificateURL: asset.GetCertificateHref(), + SbomURL: asset.GetSbomHref(), + } + } + + // Transform manifest images: extract ref, digest, add platform from map key + images := make(map[string]*ReleaseImage) + for platform, image := range manifest.GetImages() { + images[platform] = &ReleaseImage{ + Ref: image.GetRef(), + Digest: image.GetDigest(), + Platform: platform, + } + } + + // Build request body + req := &RecordReleaseRequest{ + Org: org, + Name: name, + Version: version, + RepositoryURL: repositoryURL, + CommitSha: commitSha, + WorkflowRunID: workflowRunID, + Documentation: documentation, + Changelog: changelog, + ConfigSchema: configSchema, + Capabilities: capabilities, + SignatureURL: manifest.GetSignatureHref(), + CertificateURL: manifest.GetCertificateHref(), + Assets: assets, + Images: images, + } + + bodyBytes, err := json.Marshal(req) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: error: marshaling request body: %v\n", err) + os.Exit(1) + } + + // POST to registry API + baseURL, err := url.Parse(registryURL) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: error: parsing registry URL: %v\n", err) + os.Exit(1) + } + endpoint := baseURL.JoinPath("/api/v1/ingest/release") + httpReq, err := http.NewRequest(http.MethodPost, endpoint.String(), bytes.NewReader(bodyBytes)) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: error: creating HTTP request: %v\n", err) + os.Exit(1) + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: &authTransport{ + token: token, + base: http.DefaultTransport, + }, + } + resp, err := client.Do(httpReq) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: error: HTTP request failed: %v\n", err) + os.Exit(1) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "record-release: error: reading response body: %v\n", err) + os.Exit(1) + } + + // Handle response codes + switch resp.StatusCode { + case http.StatusOK: + result, _ := json.Marshal(map[string]interface{}{ + "status": "success", + "code": http.StatusOK, + "version": version, + }) + fmt.Println(string(result)) + case http.StatusConflict: + // 409 = already exists, not an error (expected during dual-write migration) + result, _ := json.Marshal(map[string]interface{}{ + "status": "already_exists", + "code": http.StatusConflict, + "version": version, + }) + fmt.Println(string(result)) + default: + fmt.Fprintf(os.Stderr, "::error::Registry API record failed: HTTP %d\n", resp.StatusCode) + fmt.Fprintf(os.Stderr, "%s\n", string(respBody)) + os.Exit(1) + } +}