From 33949207671e97fedc0a5aecf0ac3ed64bbb884f Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Thu, 3 Jul 2025 11:46:55 +0200 Subject: [PATCH 01/13] ci: update GitHub Actions workflow for monorepo structure - Remove unnecessary submodule update job (no longer needed in monorepo) - Update service matrix to include all services (admin-backend, document-extractor, rag-backend, mcp-server) - Add separate job for library testing (rag-core-lib, rag-core-api, admin-api-lib, extractor-api-lib) - Fix Docker build paths to use services/ prefix for service Dockerfiles - Update library builds to use libs/Dockerfile with proper DIRECTORY and TEST args - Replace deprecated ::set-output with modern GITHUB_OUTPUT - Remove submodule-related checkout options - Ensure CI properly tests both services and libraries in the monorepo --- .github/workflows/lint-and-test.yml | 62 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 45801c28..664e9579 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -22,64 +22,60 @@ jobs: if [[ -z "$SANITIZED_REF" || "$SANITIZED_REF" =~ ^-+$ ]]; then SANITIZED_REF="tmp-branch" fi - echo "::set-output name=sanitized_ref::${SANITIZED_REF}" + echo "sanitized_ref=${SANITIZED_REF}" >> $GITHUB_OUTPUT shell: bash env: GITHUB_HEAD_REF: ${{ github.head_ref }} - update-submodules: - name: update-submodules + build-and-lint-services: + name: Build and Lint Services runs-on: ubuntu-latest - permissions: - contents: write + needs: [sanitize-branch-name] + strategy: + fail-fast: true + matrix: + service: [ "admin-backend", "document-extractor", "rag-backend", "mcp-server"] steps: - - name: checkout + - name: Checkout code uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.ref }} - submodules: false - - name: update-submodules - id: update-submodules + - name: Set Docker Image Name run: | - git config --global url."https://github.com/".insteadOf "git@github.com:" - git submodule sync --recursive - git submodule update --init --remote --recursive + echo "IMAGE_NAME=${{ matrix.service }}:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV + shell: bash - if [ -n "$(git status --porcelain)" ]; then - echo "Submodules have new commits. Committing and pushing..." - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git commit -am "chore: update submodules to latest main" - git push - else - echo "No submodule changes." - fi + - name: Build lint image + run: | + docker build -t $IMAGE_NAME --build-arg dev=1 -f services/${{ matrix.service }}/Dockerfile . - build-and-lint: - name: Build and Lint + - name: Generate lint report + run: | + docker run --rm --entrypoint make "$IMAGE_NAME" lint + + - name: Run tests + run: | + docker run --rm --entrypoint make "$IMAGE_NAME" test + + build-and-lint-libs: + name: Build and Lint Libraries runs-on: ubuntu-latest - needs: [sanitize-branch-name, update-submodules] + needs: [sanitize-branch-name] strategy: fail-fast: true matrix: - service: [ "admin-backend", "document-extractor", "rag-backend"] + library: [ "rag-core-lib", "rag-core-api", "admin-api-lib", "extractor-api-lib"] steps: - name: Checkout code uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.ref }} - submodules: true - name: Set Docker Image Name run: | - echo "IMAGE_NAME=${{ matrix.service }}:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV + echo "IMAGE_NAME=${{ matrix.library }}:${{ needs.sanitize-branch-name.outputs.sanitized_ref }}-${{ github.run_number }}" >> $GITHUB_ENV shell: bash - name: Build lint image run: | - docker build -t $IMAGE_NAME --build-arg dev=1 --build-arg TAG=debug -f ${{ matrix.service }}/Dockerfile . + docker build -t $IMAGE_NAME --build-arg DIRECTORY=${{ matrix.library }} --build-arg TEST=1 -f libs/Dockerfile . - name: Generate lint report run: | From 912dae7cbf921010cb4dd4ef91c1af7f2836ea70 Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Thu, 3 Jul 2025 11:49:47 +0200 Subject: [PATCH 02/13] ci: align GitHub Actions with Tiltfile patterns - Fix library build context to use 'libs' instead of root directory - Use TEST=0 for library linting (matching Tiltfile pattern) - Use separate images for library lint vs test (TEST=0 vs TEST=1 with DIRECTORY) - Remove unnecessary --entrypoint overrides (Dockerfile already sets correct entrypoint) - Ensure CI builds match exactly how Tilt builds work locally --- .github/workflows/lint-and-test.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 664e9579..8d61b619 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -50,11 +50,11 @@ jobs: - name: Generate lint report run: | - docker run --rm --entrypoint make "$IMAGE_NAME" lint + docker run --rm $IMAGE_NAME make lint - name: Run tests run: | - docker run --rm --entrypoint make "$IMAGE_NAME" test + docker run --rm $IMAGE_NAME make test build-and-lint-libs: name: Build and Lint Libraries @@ -75,12 +75,16 @@ jobs: - name: Build lint image run: | - docker build -t $IMAGE_NAME --build-arg DIRECTORY=${{ matrix.library }} --build-arg TEST=1 -f libs/Dockerfile . + docker build -t $IMAGE_NAME --build-arg TEST=0 -f libs/Dockerfile libs - name: Generate lint report run: | - docker run --rm --entrypoint make "$IMAGE_NAME" lint + docker run --rm $IMAGE_NAME make lint + + - name: Build test image + run: | + docker build -t $IMAGE_NAME-test --build-arg DIRECTORY=${{ matrix.library }} --build-arg TEST=1 -f libs/Dockerfile libs - name: Run tests run: | - docker run --rm --entrypoint make "$IMAGE_NAME" test + docker run --rm $IMAGE_NAME-test make test From df2fcfbaa59c59de9f71f32a79a320bb666685b7 Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Thu, 10 Jul 2025 14:11:09 +0200 Subject: [PATCH 03/13] fix: rename document_reference_id_get method to document_reference (#40) --- libs/admin-api-lib/src/admin_api_lib/impl/admin_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/admin_api.py b/libs/admin-api-lib/src/admin_api_lib/impl/admin_api.py index 4ecdd4c5..aabe1dfd 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/admin_api.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/admin_api.py @@ -141,7 +141,7 @@ async def upload_file( await file_uploader.upload_file(str(request.base_url), file) @inject - async def document_reference_id_get( + async def document_reference( self, identification: str, document_reference_retriever: DocumentReferenceRetriever = Depends( From 01227cb6fa91dc6ae0d660b83726bdbfd3464a18 Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Thu, 10 Jul 2025 14:12:17 +0200 Subject: [PATCH 04/13] refactor: clean up imports and fix minor issues in chat-related components (#41) --- .../frontend/libs/chat-app/data-access/+state/chat.store.ts | 6 +++--- .../libs/chat-app/models/chat-document-group.model.ts | 2 +- .../frontend/libs/chat-app/models/chat-message.model.ts | 2 +- services/frontend/libs/chat-app/ui/ChatBubble.vue | 3 ++- services/frontend/libs/chat-app/ui/ChatDocumentGroup.vue | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/services/frontend/libs/chat-app/data-access/+state/chat.store.ts b/services/frontend/libs/chat-app/data-access/+state/chat.store.ts index 9aec0c49..b30986a3 100644 --- a/services/frontend/libs/chat-app/data-access/+state/chat.store.ts +++ b/services/frontend/libs/chat-app/data-access/+state/chat.store.ts @@ -1,6 +1,4 @@ import { newUid } from "@shared/utils"; -import { InformationPiece } from "libs/chat-app/models/chat-response.model"; -import { DocumentResponseModel } from "libs/chat-app/models/document-response.model"; import { marked } from "marked"; import { defineStore } from 'pinia'; import { computed, ref } from 'vue'; @@ -8,10 +6,12 @@ import { useI18n } from 'vue-i18n'; import { ChatDocumentModel, mapToChatDocuments } from "../../models/chat-document.model"; import { ChatMessageModel } from "../../models/chat-message.model"; import { ChatRequestModel, mapToChatRequestModel } from "../../models/chat-request.model"; +import { InformationPiece } from "../../models/chat-response.model"; +import { DocumentResponseModel } from "../../models/document-response.model"; import { ChatAPI } from "../chat.api"; export const useChatStore = defineStore('chat', () => { - const {t, locale} = useI18n(); + const {t} = useI18n(); const conversationId = ref(); const chatHistory = ref([]); const chatDocuments = ref([]); diff --git a/services/frontend/libs/chat-app/models/chat-document-group.model.ts b/services/frontend/libs/chat-app/models/chat-document-group.model.ts index 6f8c2361..e54d6a92 100644 --- a/services/frontend/libs/chat-app/models/chat-document-group.model.ts +++ b/services/frontend/libs/chat-app/models/chat-document-group.model.ts @@ -1,4 +1,4 @@ export interface ChatDocumentGroupModel { title: string; source: string; -}; +} diff --git a/services/frontend/libs/chat-app/models/chat-message.model.ts b/services/frontend/libs/chat-app/models/chat-message.model.ts index 7d4d8887..443d725c 100644 --- a/services/frontend/libs/chat-app/models/chat-message.model.ts +++ b/services/frontend/libs/chat-app/models/chat-message.model.ts @@ -7,5 +7,5 @@ export interface ChatMessageModel { dateTime?: Date; role: ChatRole; hasError?: boolean; - skipAPI?: Boolean; + skipAPI?: boolean; } \ No newline at end of file diff --git a/services/frontend/libs/chat-app/ui/ChatBubble.vue b/services/frontend/libs/chat-app/ui/ChatBubble.vue index 3e3a4ab5..9146fba5 100644 --- a/services/frontend/libs/chat-app/ui/ChatBubble.vue +++ b/services/frontend/libs/chat-app/ui/ChatBubble.vue @@ -1,7 +1,7 @@