Taskfile.yml

version: "3"

vars:
  MINIKUBE_MEM: 8G
  MINIKUBE_CPU: 2
  DAGSTER_CHART_VERSION: 1.8.7
  DAGSTER_WEBSERVER_PORT: 8181
  OPEN_METADATA_CHART_VERSION: 1.5.4
  OPEN_METADATA_UI_PORT: 8585
  CODE_LOCATION_MODERATE_LOCAL_NAME: moderate-dagster
  CODE_LOCATION_MODERATE_REMOTE_NAME: localhost:5000/moderate-dagster
  CODE_LOCATION_MODERATE_REMOTE_TAG: latest
  VENV_NAME: .venv
  SECRETS_ENV_FILE: dagster-secrets.env
  SECRETS_NAME: dagster-secrets
  VALUES_DAGSTER_TMPL: values-dagster.yaml.template
  VALUES_DAGSTER: values-dagster.yaml
  VALUES_OPEN_METADATA_TMPL: values-openmetadata.yaml.template
  VALUES_OPEN_METADATA: values-openmetadata.yaml
  OPEN_METADATA_PRIVKEY: om_privkey.der
  OPEN_METADATA_PUBKEY: om_pubkey.der
  OPEN_METADATA_PEM_PRIVKEY: om_privkey.pem
  MODERATE_CLI_IMAGE: docker.io/agmangas/moderate-cli:0.5.1
  KEYCLOAK_INIT_SLEEP_SECONDS: 40
  MAIN_IP_ADDRESS:
    sh: ip route get 1.2.3.4 | awk '{print $7}'
  OPEN_METADATA_ROOT_URL: http://{{.MAIN_IP_ADDRESS}}:8585
  KEYCLOAK_URL: http://{{.MAIN_IP_ADDRESS}}:8080
  REGISTRY_TUNNEL_CONTAINER_NAME: minikube-registry-tunnel

env:
  CODE_LOCATION_MODERATE_REMOTE_NAME: "{{.CODE_LOCATION_MODERATE_REMOTE_NAME}}"
  CODE_LOCATION_MODERATE_REMOTE_TAG: "{{.CODE_LOCATION_MODERATE_REMOTE_TAG}}"
  OPEN_METADATA_ROOT_URL: "{{.OPEN_METADATA_ROOT_URL}}"
  KEYCLOAK_URL: "{{.KEYCLOAK_URL}}"

dotenv: [.env, .env.default]

tasks:
  clean:
    desc: Clean up the dev environment
    cmds:
      - minikube delete
      - docker rm -f {{.REGISTRY_TUNNEL_CONTAINER_NAME}}
      - docker compose -f {{.ROOT_DIR}}/docker-compose.dev.yml down -v
      - cmd: rm {{.ROOT_DIR}}/{{.VALUES_DAGSTER}}
        ignore_error: true
      - cmd: rm {{.ROOT_DIR}}/{{.VALUES_OPEN_METADATA}}
        ignore_error: true
      - cmd: rm {{.ROOT_DIR}}/{{.SECRETS_ENV_FILE}}
        ignore_error: true
      - cmd: rm {{.ROOT_DIR}}/{{.OPEN_METADATA_PRIVKEY}}
        ignore_error: true
      - cmd: rm {{.ROOT_DIR}}/{{.OPEN_METADATA_PUBKEY}}
        ignore_error: true
      - cmd: rm {{.ROOT_DIR}}/{{.OPEN_METADATA_PEM_PRIVKEY}}
        ignore_error: true

  show-env:
    silent: true
    cmds:
      - for:
          - MAIN_IP_ADDRESS
          - OPEN_METADATA_ROOT_URL
          - KEYCLOAK_URL
          - OPEN_METADATA_ROOT_URL
          - KEYCLOAK_URL
        cmd: 'echo -e "{{.ITEM}} = {{index . .ITEM}}"'

  moderate-cli-init-keycloak:
    desc: Create the Keycloak entities using the MODERATE CLI
    vars:
      KEYCLOAK_ADMIN_USER: "{{.KEYCLOAK_ADMIN}}"
      KEYCLOAK_ADMIN_PASS: "{{.KEYCLOAK_ADMIN_PASSWORD}}"
      APISIX_CLIENT_SECRET: secret
    cmds:
      - >
        docker run --rm -it
        -e KEYCLOAK_ADMIN_USER={{.KEYCLOAK_ADMIN_USER}}
        -e KEYCLOAK_ADMIN_PASS={{.KEYCLOAK_ADMIN_PASS}}
        -e KEYCLOAK_URL={{.KEYCLOAK_URL}}
        -e APISIX_CLIENT_SECRET={{.APISIX_CLIENT_SECRET}}
        -e OPEN_METADATA_CLIENT_ID={{.OPEN_METADATA_CLIENT_ID}}
        -e OPEN_METADATA_ROOT_URL={{.OPEN_METADATA_ROOT_URL}}
        -e MODERATE_REALM={{.MODERATE_REALM}}
        {{.MODERATE_CLI_IMAGE}} /bin/bash -c
        "moderatecli create-keycloak-realm &&
        moderatecli create-apisix-client &&
        moderatecli create-open-metadata-client"

  compose-up:
    desc: Start the Compose stack with the dependency services for development
    cmds:
      - docker compose -f {{.ROOT_DIR}}/docker-compose.dev.yml up -d --build --wait
      - echo "🕐 Waiting for Keycloak to start..."
      - sleep {{.KEYCLOAK_INIT_SLEEP_SECONDS}}
      - task: moderate-cli-init-keycloak

  minikube-start:
    desc: Create the Minikube K8s cluster
    vars:
      TUNNEL_SLEEP_SECONDS: 40
    cmds:
      - minikube start --memory={{.MINIKUBE_MEM}} --cpus={{.MINIKUBE_CPU}}
      - minikube addons enable registry
      # https://minikube.sigs.k8s.io/docs/handbook/registry
      - >
        docker run -d
        --name {{.REGISTRY_TUNNEL_CONTAINER_NAME}}
        --network=host
        alpine ash -c "apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:$(minikube ip):5000"
      - echo "🕐 Waiting for Minikube registry tunnel..."
      - sleep {{.TUNNEL_SLEEP_SECONDS}}
    status:
      - minikube status

  gen-open-metadata-keypair:
    desc: Generate the Open Metadata keypair
    cmds:
      - openssl genrsa -out {{.ROOT_DIR}}/{{.OPEN_METADATA_PEM_PRIVKEY}} 2048
      - >
        openssl pkcs8 -topk8 -inform PEM -outform DER
        -in {{.ROOT_DIR}}/{{.OPEN_METADATA_PEM_PRIVKEY}}
        -out {{.ROOT_DIR}}/{{.OPEN_METADATA_PRIVKEY}} -nocrypt
      - >
        openssl rsa
        -in {{.ROOT_DIR}}/{{.OPEN_METADATA_PEM_PRIVKEY}}
        -pubout -outform DER
        -out {{.ROOT_DIR}}/{{.OPEN_METADATA_PUBKEY}}
    status:
      - test -f {{.ROOT_DIR}}/{{.OPEN_METADATA_PRIVKEY}}

  build-dagster-secrets-envfile:
    desc: Build the Dagster secrets envfile
    vars:
      SECRETS_PATH: "{{.ROOT_DIR}}/{{.SECRETS_ENV_FILE}}"
    cmds:
      - cmd: rm {{.ROOT_DIR}}/{{.SECRETS_ENV_FILE}}
        ignore_error: true
      - echo "KEYCLOAK_SERVER_URL=http://host.docker.internal:8080/" >> {{.SECRETS_PATH}}
      - echo "KEYCLOAK_ADMIN_USERNAME={{.KEYCLOAK_ADMIN}}" >> {{.SECRETS_PATH}}
      - echo "KEYCLOAK_ADMIN_PASSWORD={{.KEYCLOAK_ADMIN_PASSWORD}}" >> {{.SECRETS_PATH}}
      - echo "POSTGRES_HOST=host.docker.internal" >> {{.SECRETS_PATH}}
      - echo "POSTGRES_PORT=5432" >> {{.SECRETS_PATH}}
      - echo "POSTGRES_USERNAME={{.POSTGRES_USER}}" >> {{.SECRETS_PATH}}
      - echo "POSTGRES_PASSWORD={{.POSTGRES_PASSWORD}}" >> {{.SECRETS_PATH}}
      - echo "OPEN_METADATA_HOST=host.docker.internal" >> {{.SECRETS_PATH}}
      - echo "OPEN_METADATA_PORT=8585" >> {{.SECRETS_PATH}}
      - echo "OPEN_METADATA_TOKEN=null" >> {{.SECRETS_PATH}}
      - echo "API_BASE_URL=http://host.docker.internal:9080" >> {{.SECRETS_PATH}}
      - echo "API_USERNAME={{.API_USERNAME}}" >> {{.SECRETS_PATH}}
      - echo "API_PASSWORD={{.API_PASSWORD}}" >> {{.SECRETS_PATH}}
      - echo "S3_ACCESS_KEY_ID={{.S3_ACCESS_KEY_ID}}" >> {{.SECRETS_PATH}}
      - echo "S3_SECRET_ACCESS_KEY={{.S3_SECRET_ACCESS_KEY}}" >> {{.SECRETS_PATH}}
      - echo "S3_REGION={{.S3_REGION}}" >> {{.SECRETS_PATH}}
      - echo "S3_BUCKET_NAME={{.S3_BUCKET_NAME}}" >> {{.SECRETS_PATH}}
      - echo "S3_ENDPOINT_URL={{.S3_ENDPOINT_URL}}" >> {{.SECRETS_PATH}}
      - echo "RABBIT_URL={{.RABBIT_URL}}" >> {{.SECRETS_PATH}}
      - echo "MATRIX_PROFILE_JOB_IMAGE={{.MATRIX_PROFILE_JOB_IMAGE}}" >> {{.SECRETS_PATH}}

  create-dagster-secrets:
    desc: Create the Dagster Kubernetes secrets
    deps:
      - build-dagster-secrets-envfile
    cmds:
      - kubectl create secret generic {{.SECRETS_NAME}} --from-env-file={{.ROOT_DIR}}/{{.SECRETS_ENV_FILE}}

  create-open-metadata-secrets:
    desc: Create the Open Metadata Kubernetes secrets
    deps:
      - gen-open-metadata-keypair
    cmds:
      - cmd: >
          kubectl create secret generic
          openmetadata-postgres-secrets
          --from-literal=openmetadata-postgres-password={{.POSTGRES_PASSWORD}}
        ignore_error: true
      - cmd: >
          kubectl create secret generic
          openmetadata-jwt-keys
          --from-file {{.ROOT_DIR}}/{{.OPEN_METADATA_PRIVKEY}}
          --from-file {{.ROOT_DIR}}/{{.OPEN_METADATA_PUBKEY}}
        ignore_error: true

  build-helm-values:
    desc: Build the Helm values file for Dagster
    env:
      ELASTIC_HOST: host.docker.internal
      ELASTIC_PORT: 9200
      POSTGRES_OPEN_METADATA_HOST: host.docker.internal
      POSTGRES_OPEN_METADATA_PORT: 5432
      POSTGRES_OPEN_METADATA_DBNAME: "{{.OPEN_METADATA_POSTGRES_DB}}"
      POSTGRES_OPEN_METADATA_USER: "{{.POSTGRES_USER}}"
      POSTGRES_OPEN_METADATA_PASSWORD: "{{.POSTGRES_PASSWORD}}"
      OPEN_METADATA_PUBKEY: "{{.OPEN_METADATA_PUBKEY}}"
      OPEN_METADATA_PRIVKEY: "{{.OPEN_METADATA_PRIVKEY}}"
    cmds:
      - envsubst < {{.ROOT_DIR}}/{{.VALUES_DAGSTER_TMPL}} > {{.ROOT_DIR}}/{{.VALUES_DAGSTER}}
      - envsubst < {{.ROOT_DIR}}/{{.VALUES_OPEN_METADATA_TMPL}} > {{.ROOT_DIR}}/{{.VALUES_OPEN_METADATA}}

  install-helm-dagster:
    desc: Install Dagster Helm chart
    deps:
      - build-helm-values
      - minikube-start
    cmds:
      - task: create-dagster-secrets
      - helm repo add dagster https://dagster-io.github.io/helm
      - >
        helm upgrade --install dagster dagster/dagster 
        --version {{.DAGSTER_CHART_VERSION}}
        -f {{.ROOT_DIR}}/{{.VALUES_DAGSTER}}

  install-helm-open-metadata:
    desc: Install Open Metadata Helm chart
    deps:
      - build-helm-values
      - minikube-start
    cmds:
      - task: create-open-metadata-secrets
      - helm repo add open-metadata https://helm.open-metadata.org/
      - >
        helm upgrade --install openmetadata open-metadata/openmetadata
        --version {{.OPEN_METADATA_CHART_VERSION}}
        -f {{.ROOT_DIR}}/{{.VALUES_OPEN_METADATA}}

  push-code-image:
    desc: Build and push a code location image to the image registry
    vars:
      requires: [LOCAL_NAME, REMOTE_NAME, REMOTE_TAG, CODE_PATH]
    cmds:
      - docker build -t {{.LOCAL_NAME}} {{.CODE_PATH}}
      - docker tag {{.LOCAL_NAME}} {{.REMOTE_NAME}}:{{.REMOTE_TAG}}
      - docker push {{.REMOTE_NAME}}:{{.REMOTE_TAG}}

  push-code-image-moderate:
    desc: Build and push the moderate code location image to the image registry
    cmds:
      - task: push-code-image
        vars:
          LOCAL_NAME: "{{.CODE_LOCATION_MODERATE_LOCAL_NAME}}"
          REMOTE_NAME: "{{.CODE_LOCATION_MODERATE_REMOTE_NAME}}"
          REMOTE_TAG: "{{.CODE_LOCATION_MODERATE_REMOTE_TAG}}"
          CODE_PATH: "{{.ROOT_DIR}}/moderate"

  start-dev-k8s:
    desc: Start the dev environment in the local Kubernetes cluster
    vars:
      WAIT_TIMEOUT_SECS: 600
      WAIT_PODS_SECS: 10
    cmds:
      - task: minikube-start
      - task: push-code-image-moderate
      - task: compose-up
      - task: install-helm-dagster
      - task: install-helm-open-metadata
      - echo "🕐 Waiting for all pods to be ready (feel free to Ctrl+C)"
      - sleep {{.WAIT_PODS_SECS}}
      - kubectl wait --for=condition=Ready pods --all --timeout={{.WAIT_TIMEOUT_SECS}}s

  forward-k8s-dagster-ui:
    desc: Forward the Dagster Webserver port
    cmds:
      - cmd: open http://{{.MAIN_IP_ADDRESS}}:{{.DAGSTER_WEBSERVER_PORT}}
        platforms: [darwin]
      - cmd: xdg-open http://{{.MAIN_IP_ADDRESS}}:{{.DAGSTER_WEBSERVER_PORT}}
        platforms: [linux]
      - >
        DAGSTER_WEBSERVER_POD_NAME=$(
        kubectl get pods --namespace default 
        -l "component=dagster-webserver" 
        -o jsonpath="{.items[0].metadata.name}"
        ) && kubectl port-forward --address=0.0.0.0
        ${DAGSTER_WEBSERVER_POD_NAME} {{.DAGSTER_WEBSERVER_PORT}}:80

  forward-k8s-open-metadata-ui:
    desc: Forward the Open Metadata UI port
    cmds:
      - cmd: open http://{{.MAIN_IP_ADDRESS}}:{{.OPEN_METADATA_UI_PORT}}
        platforms: [darwin]
      - cmd: xdg-open http://{{.MAIN_IP_ADDRESS}}:{{.OPEN_METADATA_UI_PORT}}
        platforms: [linux]
      - >
        OPEN_METADATA_UI_POD_NAME=$(
        kubectl get pods --namespace default
        -l "app.kubernetes.io/name=openmetadata,app.kubernetes.io/instance=openmetadata"
        -o jsonpath="{.items[0].metadata.name}"
        ) && kubectl port-forward --address=0.0.0.0
        ${OPEN_METADATA_UI_POD_NAME} {{.OPEN_METADATA_UI_PORT}}:8585

  create-venv-dagster-dev:
    desc: Create the Dagster dev virtualenv
    cmds:
      - virtualenv {{.ROOT_DIR}}/{{.VENV_NAME}}
      - "{{.ROOT_DIR}}/{{.VENV_NAME}}/bin/pip install -U -e {{.ROOT_DIR}}/moderate[dev]"

  dagster-dev:
    desc: Run the Dagster dev CLI
    deps:
      - compose-up
      - create-venv-dagster-dev
    env:
      KEYCLOAK_SERVER_URL: http://localhost:8080/
      KEYCLOAK_ADMIN_USERNAME: "{{.KEYCLOAK_ADMIN}}"
      KEYCLOAK_ADMIN_PASSWORD: "{{.KEYCLOAK_ADMIN_PASSWORD}}"
      POSTGRES_HOST: localhost
      POSTGRES_PORT: 5432
      POSTGRES_USERNAME: "{{.POSTGRES_USER}}"
      POSTGRES_PASSWORD: "{{.POSTGRES_PASSWORD}}"
      OPEN_METADATA_HOST: localhost
      OPEN_METADATA_PORT: 8585
      API_BASE_URL: http://localhost:9080
      API_USERNAME: "{{.API_USERNAME}}"
      API_PASSWORD: "{{.API_PASSWORD}}"
      S3_ACCESS_KEY_ID: "{{.S3_ACCESS_KEY_ID}}"
      S3_SECRET_ACCESS_KEY: "{{.S3_SECRET_ACCESS_KEY}}"
      S3_REGION: "{{.S3_REGION}}"
      S3_BUCKET_NAME: "{{.S3_BUCKET_NAME}}"
      S3_ENDPOINT_URL: "{{.S3_ENDPOINT_URL}}"
      RABBIT_URL: "{{.RABBIT_URL}}"
      MATRIX_PROFILE_JOB_IMAGE: "{{.MATRIX_PROFILE_JOB_IMAGE}}"
    cmds:
      - cd {{.ROOT_DIR}}/moderate && {{.ROOT_DIR}}/{{.VENV_NAME}}/bin/dagster dev

  update-k8s-open-metadata-token:
    desc: Update the Open Metadata token in the Dagster Kubernetes secret
    requires:
      vars: [OPEN_METADATA_TOKEN]
    env:
      SECRET_NAME: "{{.SECRETS_NAME}}"
      NAMESPACE: default
      DAGSTER_RELEASE: dagster
      DAGSTER_CHART: dagster/dagster
    cmds:
      - "{{.ROOT_DIR}}/scripts/update-open-metadata-secret.sh"
    preconditions:
      - context=$(kubectl config current-context) && [[ $context == "minikube" ]]

  force-restart-k8s-dagster:
    desc: Force restart the Dagster deployment in the local Kubernetes cluster
    vars:
      USER_DEPLOYMENT:
        sh: kubectl get deployments -A -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep dagster-user-deployments
    cmds:
      - task: push-code-image-moderate
      - kubectl rollout restart deployment/{{.USER_DEPLOYMENT}}

  format:
    desc: Format Python code using black
    cmds:
      - "{{.ROOT_DIR}}/{{.VENV_NAME}}/bin/black {{.ROOT_DIR}}/moderate/moderate {{.ROOT_DIR}}/moderate/moderate_tests"

  lint:
    desc: Lint Python code using ruff
    cmds:
      - "{{.ROOT_DIR}}/{{.VENV_NAME}}/bin/ruff check {{.ROOT_DIR}}/moderate/moderate {{.ROOT_DIR}}/moderate/moderate_tests"

  lint-fix:
    desc: Lint and auto-fix Python code using ruff
    cmds:
      - "{{.ROOT_DIR}}/{{.VENV_NAME}}/bin/ruff check --fix {{.ROOT_DIR}}/moderate/moderate {{.ROOT_DIR}}/moderate/moderate_tests"