Merge staging into prod

.github/workflows/deploy-docs-prod.yaml

@@ -38,21 +38,37 @@ jobs:
         with:
           repository: validmind/validmind-library
           path: site/_source/validmind-library
-          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
 
       - name: Check out installation repository
         uses: actions/checkout@v4
         with:
           repository: validmind/installation
           path: site/_source/installation
-          token: ${{ secrets.INSTALLATION_RO_PAT }}
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
+          sparse-checkout: |
+            site/installation
+          sparse-checkout-cone-mode: true
 
       - name: Check out release-notes repository
         uses: actions/checkout@v4
         with:
           repository: validmind/release-notes
           path: site/_source/release-notes
-          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
+          sparse-checkout: |
+            releases
+          sparse-checkout-cone-mode: true
+
+      - name: Check out backend repository
+        uses: actions/checkout@v4
+        with:
+          repository: validmind/backend
+          path: site/_source/backend
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
+          sparse-checkout: |
+            src/backend/templates/documentation/model_documentation
+          sparse-checkout-cone-mode: true
 
       - name: Set up Quarto
         uses: quarto-dev/quarto-actions/setup@v2
@@ -73,6 +89,11 @@ jobs:
           mkdir -p validmind
           rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
 
+      - name: Generate template schema docs
+        run: |
+          pip install json-schema-for-humans
+          BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py
+
       - name: Populate installation
         run: cp -r site/_source/installation/site/installation site/installation
 
@@ -96,7 +117,7 @@ jobs:
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} && aws configure set default.region us-east-1
 
       - name: Deploy docs prod site
-        run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager
+        run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager
 
       # Release headroom and shrink before final lightweight steps & post-job
       - name: Release reserve & shrink
@@ -107,6 +128,7 @@ jobs:
           remove_paths: |
             site/_source/installation
             site/_source/release-notes
+            site/_source/backend
             site/render_errors.log
             site/_freeze
             dev.env

.github/workflows/deploy-docs-staging.yaml

@@ -38,22 +38,38 @@ jobs:
         with:
           repository: validmind/validmind-library
           path: site/_source/validmind-library
-          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
 
       - name: Check out installation repository
         uses: actions/checkout@v4
         with:
           repository: validmind/installation
           path: site/_source/installation
-          token: ${{ secrets.INSTALLATION_RO_PAT }}
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
+          sparse-checkout: |
+            site/installation
+          sparse-checkout-cone-mode: true
 
       - name: Check out release-notes repository
         uses: actions/checkout@v4
         with:
           repository: validmind/release-notes
           path: site/_source/release-notes
-          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
-
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
+          sparse-checkout: |
+            releases
+          sparse-checkout-cone-mode: true
+
+      - name: Check out backend repository
+        uses: actions/checkout@v4
+        with:
+          repository: validmind/backend
+          path: site/_source/backend
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
+          sparse-checkout: |
+            src/backend/templates/documentation/model_documentation
+          sparse-checkout-cone-mode: true
+
       - name: Set up Quarto
         uses: quarto-dev/quarto-actions/setup@v2
         with:
@@ -72,7 +88,12 @@ jobs:
           rm -rf validmind
           mkdir -p validmind
           rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
-
+
+      - name: Generate template schema docs
+        run: |
+          pip install json-schema-for-humans
+          BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py
+
       - name: Populate installation
         run: cp -r site/_source/installation/site/installation site/installation
 
@@ -99,7 +120,7 @@ jobs:
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-west-2
 
       - name: Deploy docs staging site
-        run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
+        run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
 
       # Release headroom and shrink before final lightweight steps & post-job
       - name: Release reserve & shrink
@@ -110,6 +131,7 @@ jobs:
           remove_paths: |
             site/_source/installation
             site/_source/release-notes
+            site/_source/backend
             site/render_errors.log
             site/_freeze
             dev.env

.github/workflows/publish-llm-markdown.yaml

@@ -0,0 +1,109 @@
+name: Publish LLM markdown
+
+on:
+  push:
+    branches:
+      - main
+      - prod
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out documentation repository
+        uses: actions/checkout@v4
+
+      - name: Free space + create reserve
+        uses: ./.github/actions/free-disk-space
+        with:
+          remove_dotnet: "true"
+          remove_android: "true"
+          remove_haskell: "true"
+          prune_docker: "true"
+          apt_cleanup: "true"
+          create_reserve_gb: "3"
+
+      - run: mkdir -p site/_source
+
+      - name: Check out validmind-library repository
+        uses: actions/checkout@v4
+        with:
+          repository: validmind/validmind-library
+          path: site/_source/validmind-library
+          token: ${{ secrets.DOCS_CI_RO_PAT }}
+
+      - name: Set up Quarto
+        uses: quarto-dev/quarto-actions/setup@v2
+        with:
+          version: ${{ vars.QUARTO_VERSION }}
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Generate Python library docs
+        run: |
+          cd _source/validmind-library
+          make install && make quarto-docs
+          cd ../../
+          rm -rf validmind
+          mkdir -p validmind
+          rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
+        working-directory: site
+
+      - name: Install pandoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc
+
+      - name: Render LLM markdown
+        run: bash llm/render.sh
+        working-directory: site
+
+      - name: Set archive name
+        id: archive
+        run: |
+          BRANCH="${GITHUB_REF_NAME//\//-}"
+          echo "name=docs-markdown-${BRANCH}.tar.gz" >> "$GITHUB_OUTPUT"
+
+      - name: Create tar.gz archive
+        run: |
+          cd site/llm/_llm-output
+          tar -czf "/tmp/${{ steps.archive.outputs.name }}" .
+
+      # main → staging bucket (us-west-2), prod → prod bucket (us-east-1)
+      - name: Configure AWS credentials
+        run: |
+          if [ "${{ github.ref_name }}" = "prod" ]; then
+            aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }}
+            aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }}
+            aws configure set default.region us-east-1
+          else
+            aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }}
+            aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }}
+            aws configure set default.region us-west-2
+          fi
+
+      - name: Upload to S3
+        run: |
+          if [ "${{ github.ref_name }}" = "prod" ]; then
+            BUCKET="s3://validmind-docs-prod/site/llm/${{ steps.archive.outputs.name }}"
+          else
+            BUCKET="s3://validmind-docs-staging/site/llm/${{ steps.archive.outputs.name }}"
+          fi
+          aws s3 cp "/tmp/${{ steps.archive.outputs.name }}" "$BUCKET"
+          echo "Uploaded to $BUCKET"
+
+      - name: Release reserve & shrink
+        if: always()
+        uses: ./.github/actions/free-disk-space
+        with:
+          release_reserve: "true"
+          remove_paths: |
+            site/_source/installation
+            site/_source/release-notes
+            site/render_errors.log
+            site/_freeze

.github/workflows/validate-docs-site.yaml

@@ -33,21 +33,37 @@ jobs:
       with:
         repository: validmind/validmind-library
         path: site/_source/validmind-library
-        token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
-        
+        token: ${{ secrets.DOCS_CI_RO_PAT }}
+
     - name: Check out installation repository
       uses: actions/checkout@v4
       with:
         repository: validmind/installation
         path: site/_source/installation
-        token: ${{ secrets.INSTALLATION_RO_PAT }}
+        token: ${{ secrets.DOCS_CI_RO_PAT }}
+        sparse-checkout: |
+          site/installation
+        sparse-checkout-cone-mode: true
 
     - name: Check out release-notes repository
       uses: actions/checkout@v4
       with:
         repository: validmind/release-notes
         path: site/_source/release-notes
-        token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+        token: ${{ secrets.DOCS_CI_RO_PAT }}
+        sparse-checkout: |
+          releases
+        sparse-checkout-cone-mode: true
+
+    - name: Check out backend repository
+      uses: actions/checkout@v4
+      with:
+        repository: validmind/backend
+        path: site/_source/backend
+        token: ${{ secrets.DOCS_CI_RO_PAT }}
+        sparse-checkout: |
+          src/backend/templates/documentation/model_documentation
+        sparse-checkout-cone-mode: true
 
     - name: Verify copyright headers
       run: |
@@ -57,7 +73,7 @@ jobs:
     - name: Set up Quarto
       uses: quarto-dev/quarto-actions/setup@v2
       with:
-        version: ${{ vars.QUARTO_VERSION }}
+        version: pre-release
 
     - name: Install Poetry
       run: |
@@ -73,6 +89,11 @@ jobs:
         mkdir -p validmind
         rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
 
+    - name: Generate template schema docs
+      run: |
+        pip install json-schema-for-humans
+        BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py
+
     - name: Populate installation
       run: cp -r site/_source/installation/site/installation site/installation
 
@@ -143,6 +164,15 @@ jobs:
             body: comment
           });
 
+    - name: Install pandoc
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pandoc
+
+    - name: Validate LLM markdown render
+      run: bash llm/render.sh && bash llm/clean.sh
+      working-directory: site
+
     # Release headroom and shrink before final lightweight steps & post-job
     - name: Release reserve & shrink
       if: always()
@@ -152,6 +182,7 @@ jobs:
         remove_paths: |
           site/_source/installation
           site/_source/release-notes
+          site/_source/backend
           site/render_errors.log
           site/_freeze
           dev.env

.gitignore

@@ -36,6 +36,11 @@ site/validmind-docs.yaml
 # Python API docs are now generated on the fly
 site/validmind
 
+# Generated template schema documentation
+site/guide/templates/_template-schema-generated.qmd
+
 # Cursor rules
 .cursor/rules/
-.cursor/skills/
+.cursor/skills/
+
+_llm-output/