validmind
diff --git a/‎.github/workflows/deploy-docs-prod.yaml‎
Lines changed: 24 additions & 2 deletions b/‎.github/workflows/deploy-docs-prod.yaml‎
Lines changed: 24 additions & 2 deletions
diff --git a/‎.github/workflows/deploy-docs-staging.yaml‎
Lines changed: 26 additions & 4 deletions b/‎.github/workflows/deploy-docs-staging.yaml‎
Lines changed: 26 additions & 4 deletions
diff --git a/‎.github/workflows/publish-llm-markdown.yaml‎
Lines changed: 109 additions & 0 deletions b/‎.github/workflows/publish-llm-markdown.yaml‎
Lines changed: 109 additions & 0 deletions
diff --git a/‎.github/workflows/validate-docs-site.yaml‎
Lines changed: 34 additions & 3 deletions b/‎.github/workflows/validate-docs-site.yaml‎
Lines changed: 34 additions & 3 deletions
diff --git a/‎.gitignore‎
Lines changed: 6 additions & 1 deletion b/‎.gitignore‎
Lines changed: 6 additions & 1 deletion
diff --git a/‎README.md‎
Lines changed: 39 additions & 3 deletions b/‎README.md‎
Lines changed: 39 additions & 3 deletions
@@ -45,14 +45,30 @@ jobs:
         with:
           repository: validmind/installation
           path: site/_source/installation
-          token: ${{ secrets.INSTALLATION_RO_PAT }}
+          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          sparse-checkout: |
+            site/installation
+          sparse-checkout-cone-mode: true
 
       - name: Check out release-notes repository
         uses: actions/checkout@v4
         with:
           repository: validmind/release-notes
           path: site/_source/release-notes
           token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          sparse-checkout: |
+            releases
+          sparse-checkout-cone-mode: true
+
+      - name: Check out backend repository
+        uses: actions/checkout@v4
+        with:
+          repository: validmind/backend
+          path: site/_source/backend
+          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          sparse-checkout: |
+            src/backend/templates/documentation/model_documentation
+          sparse-checkout-cone-mode: true
 
       - name: Set up Quarto
         uses: quarto-dev/quarto-actions/setup@v2
@@ -73,6 +89,11 @@ jobs:
           mkdir -p validmind
           rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
 
+      - name: Generate template schema docs
+        run: |
+          pip install json-schema-for-humans
+          BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py
+
       - name: Populate installation
         run: cp -r site/_source/installation/site/installation site/installation
 
@@ -96,7 +117,7 @@ jobs:
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} && aws configure set default.region us-east-1
 
       - name: Deploy docs prod site
-        run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager
+        run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager
 
       # Release headroom and shrink before final lightweight steps & post-job
       - name: Release reserve & shrink
@@ -107,6 +128,7 @@ jobs:
           remove_paths: |
             site/_source/installation
             site/_source/release-notes
+            site/_source/backend
             site/render_errors.log
             site/_freeze
             dev.env
 
@@ -45,15 +45,31 @@ jobs:
         with:
           repository: validmind/installation
           path: site/_source/installation
-          token: ${{ secrets.INSTALLATION_RO_PAT }}
+          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          sparse-checkout: |
+            site/installation
+          sparse-checkout-cone-mode: true
 
       - name: Check out release-notes repository
         uses: actions/checkout@v4
         with:
           repository: validmind/release-notes
           path: site/_source/release-notes
           token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
-  
+          sparse-checkout: |
+            releases
+          sparse-checkout-cone-mode: true
+
+      - name: Check out backend repository
+        uses: actions/checkout@v4
+        with:
+          repository: validmind/backend
+          path: site/_source/backend
+          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+          sparse-checkout: |
+            src/backend/templates/documentation/model_documentation
+          sparse-checkout-cone-mode: true
+
       - name: Set up Quarto
         uses: quarto-dev/quarto-actions/setup@v2
         with:
@@ -72,7 +88,12 @@ jobs:
           rm -rf validmind
           mkdir -p validmind
           rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
-  
+
+      - name: Generate template schema docs
+        run: |
+          pip install json-schema-for-humans
+          BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py
+
       - name: Populate installation
         run: cp -r site/_source/installation/site/installation site/installation
 
@@ -99,7 +120,7 @@ jobs:
         run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-west-2
 
       - name: Deploy docs staging site
-        run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
+        run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager
 
       # Release headroom and shrink before final lightweight steps & post-job
       - name: Release reserve & shrink
@@ -110,6 +131,7 @@ jobs:
           remove_paths: |
             site/_source/installation
             site/_source/release-notes
+            site/_source/backend
             site/render_errors.log
             site/_freeze
             dev.env
 
@@ -0,0 +1,109 @@
+name: Publish LLM markdown
+
+on:
+  push:
+    branches:
+      - main
+      - prod
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out documentation repository
+        uses: actions/checkout@v4
+
+      - name: Free space + create reserve
+        uses: ./.github/actions/free-disk-space
+        with:
+          remove_dotnet: "true"
+          remove_android: "true"
+          remove_haskell: "true"
+          prune_docker: "true"
+          apt_cleanup: "true"
+          create_reserve_gb: "3"
+
+      - run: mkdir -p site/_source
+
+      - name: Check out validmind-library repository
+        uses: actions/checkout@v4
+        with:
+          repository: validmind/validmind-library
+          path: site/_source/validmind-library
+          token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+
+      - name: Set up Quarto
+        uses: quarto-dev/quarto-actions/setup@v2
+        with:
+          version: ${{ vars.QUARTO_VERSION }}
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Generate Python library docs
+        run: |
+          cd _source/validmind-library
+          make install && make quarto-docs
+          cd ../../
+          rm -rf validmind
+          mkdir -p validmind
+          rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
+        working-directory: site
+
+      - name: Install pandoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc
+
+      - name: Render LLM markdown
+        run: bash llm/render.sh
+        working-directory: site
+
+      - name: Set archive name
+        id: archive
+        run: |
+          BRANCH="${GITHUB_REF_NAME//\//-}"
+          echo "name=docs-markdown-${BRANCH}.tar.gz" >> "$GITHUB_OUTPUT"
+
+      - name: Create tar.gz archive
+        run: |
+          cd site/llm/_llm-output
+          tar -czf "/tmp/${{ steps.archive.outputs.name }}" .
+
+      # main → staging bucket (us-west-2), prod → prod bucket (us-east-1)
+      - name: Configure AWS credentials
+        run: |
+          if [ "${{ github.ref_name }}" = "prod" ]; then
+            aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }}
+            aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }}
+            aws configure set default.region us-east-1
+          else
+            aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }}
+            aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }}
+            aws configure set default.region us-west-2
+          fi
+
+      - name: Upload to S3
+        run: |
+          if [ "${{ github.ref_name }}" = "prod" ]; then
+            BUCKET="s3://validmind-docs-prod/site/llm/${{ steps.archive.outputs.name }}"
+          else
+            BUCKET="s3://validmind-docs-staging/site/llm/${{ steps.archive.outputs.name }}"
+          fi
+          aws s3 cp "/tmp/${{ steps.archive.outputs.name }}" "$BUCKET"
+          echo "Uploaded to $BUCKET"
+
+      - name: Release reserve & shrink
+        if: always()
+        uses: ./.github/actions/free-disk-space
+        with:
+          release_reserve: "true"
+          remove_paths: |
+            site/_source/installation
+            site/_source/release-notes
+            site/render_errors.log
+            site/_freeze
@@ -34,20 +34,36 @@ jobs:
         repository: validmind/validmind-library
         path: site/_source/validmind-library
         token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
-        
+
     - name: Check out installation repository
       uses: actions/checkout@v4
       with:
         repository: validmind/installation
         path: site/_source/installation
-        token: ${{ secrets.INSTALLATION_RO_PAT }}
+        token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+        sparse-checkout: |
+          site/installation
+        sparse-checkout-cone-mode: true
 
     - name: Check out release-notes repository
       uses: actions/checkout@v4
       with:
         repository: validmind/release-notes
         path: site/_source/release-notes
         token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+        sparse-checkout: |
+          releases
+        sparse-checkout-cone-mode: true
+
+    - name: Check out backend repository
+      uses: actions/checkout@v4
+      with:
+        repository: validmind/backend
+        path: site/_source/backend
+        token: ${{ secrets.RELEASE_NOTES_RO_PAT }}
+        sparse-checkout: |
+          src/backend/templates/documentation/model_documentation
+        sparse-checkout-cone-mode: true
 
     - name: Verify copyright headers
       run: |
@@ -57,7 +73,7 @@ jobs:
     - name: Set up Quarto
       uses: quarto-dev/quarto-actions/setup@v2
       with:
-        version: ${{ vars.QUARTO_VERSION }}
+        version: pre-release
 
     - name: Install Poetry
       run: |
@@ -73,6 +89,11 @@ jobs:
         mkdir -p validmind
         rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/
 
+    - name: Generate template schema docs
+      run: |
+        pip install json-schema-for-humans
+        BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py
+
     - name: Populate installation
       run: cp -r site/_source/installation/site/installation site/installation
 
@@ -143,6 +164,15 @@ jobs:
             body: comment
           });
 
+    - name: Install pandoc
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pandoc
+
+    - name: Validate LLM markdown render
+      run: bash llm/render.sh && bash llm/clean.sh
+      working-directory: site
+
     # Release headroom and shrink before final lightweight steps & post-job
     - name: Release reserve & shrink
       if: always()
@@ -152,6 +182,7 @@ jobs:
         remove_paths: |
           site/_source/installation
           site/_source/release-notes
+          site/_source/backend
           site/render_errors.log
           site/_freeze
           dev.env
 
@@ -36,6 +36,11 @@ site/validmind-docs.yaml
 # Python API docs are now generated on the fly
 site/validmind
 
+# Generated template schema documentation
+site/guide/templates/_template-schema-generated.qmd
+
 # Cursor rules
 .cursor/rules/
-.cursor/skills/
+.cursor/skills/
+
+_llm-output/
@@ -25,11 +25,17 @@ You need:
 - To use Cursor to author documentation, the [`create-user-documentation`](https://github.com/validmind/skills/tree/main/create-user-documentation) skill
 - For Windows operating systems, install the `make` command via [Cygwin](https://cygwin.com/install.html)
 
-### Additional dependencies
+### Fetching required repositories
 
-Some interactive tables, such as our breaking changes and dependency history rely you have R and some R packages installed in order for you to be able to preview or render certain pages of the docs site locally.
+Before previewing or rendering the docs site, run:
+
+```bash
+cd site
+make get-source
+```
+
+This make action clones required repositories and generates documentation from source. Without this step, Quarto will return warnings or errors related to our installation guides, release notes, library docs, and template schema reference.
 
-**Refer to the [Breaking changes and deprecations](site/releases/breaking-changes/README.md) guide** for more information on how to install R and set up these tables.
 
 ## How to contribute
 
@@ -138,6 +144,36 @@ These directories may have sub-directories depending on their size and grouped s
 - `notebooks` — This is where notebooks retrieved from the [`validmind-library` repo](https://github.com/validmind/validmind-library) live.
 - `tests` — This is where test descriptions generated from the Python source in the [`validmind-library` repo](https://github.com/validmind/validmind-library) live.
 
+### Generated documentation
+
+Some documentation content is auto-generated from backend source files. These scripts ensure the docs stay in sync with the codebase.
+
+#### Template schema documentation
+
+The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema. CI workflows generate this automatically, but you can also regenerate locally:
+
+```bash
+cd site
+make get-source  # Clones all repos including backend (sparse checkout)
+make template-schema-docs  # Generates the schema docs
+```
+
+The backend repo is sparse-cloned as part of `make clone` (only the schema directory is fetched). To use a specific backend branch:
+
+```bash
+make clone BACKEND_BRANCH=feature-branch
+make template-schema-docs
+```
+
+**Requirements:**
+- Python 3.9+
+- SSH access to the backend repository
+
+The script reads from:
+- `backend/src/backend/templates/documentation/model_documentation/mdd_template_schema_v5.json` — template schema definition
+
+Output: `site/guide/templates/_template-schema-generated.qmd`
+
 #### Stylesheet organization (IN PROGRESS)
 
 The site uses a modular stylesheet architecture to maintain organized and maintainable styles: