Add support for iceberg audit logs for lakeFS Enterprise

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+# 1.9.2
+:new: What's new:
+- Update lakeFS version to [1.82.0](https://changelog.lakefs.io/changelog/releases/v1.82.0/)
+- Add audit log maintenance CronJob support (Enterprise-only). Runs compaction, snapshot expiration, orphan cleanup, and lakeFS commit on a configurable schedule (default: every hour). Enable with `auditLog.enabled: true` and `auditLog.maintenance: true`.
+
 # 1.9.1
 :new: What's new:
 - Update lakeFS version to [1.81.0](https://changelog.lakefs.io/changelog/releases/v1.81.0/)

charts/lakefs/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: lakefs
 description: A Helm chart for running LakeFS on Kubernetes
 type: application
-version: 1.9.1
-appVersion: 1.81.0
+version: 1.9.2
+appVersion: 1.82.0
 
 home: https://lakefs.io
 icon: https://lakefs.io/wp-content/uploads/2020/07/lake-fs-color-2.svg

charts/lakefs/templates/_env.tpl

@@ -98,6 +98,10 @@ env:
   {{- end }}
   {{- end }}
 
+  {{- if (.Values.auditLog).enabled }}
+  - name: LAKEFS_AUDIT_LOG_MAINTENANCE_ENABLED
+    value: {{ .Values.auditLog.maintenance.inProcess | quote }}
+  {{- end }}
   {{- if .Values.s3Fallback.enabled }}
   - name: LAKEFS_GATEWAYS_S3_FALLBACK_URL
     value: http://localhost:7001

charts/lakefs/templates/_helpers.tpl

@@ -132,6 +132,36 @@ app.kubernetes.io/component: replication
 app: {{ include "lakefs.name" . }}-replication
 {{- end }}
 
+{{/*
+Audit maintenance resource full name
+*/}}
+{{- define "audit.fullname" -}}
+{{- $name := include "lakefs.fullname" . }}
+{{- printf "%s-audit-maintain" $name | trunc 63 }}
+{{- end }}
+
+{{/*
+Audit maintenance common labels
+*/}}
+{{- define "audit.labels" -}}
+helm.sh/chart: {{ include "lakefs.chart" . }}
+{{ include "audit.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Audit maintenance selector labels
+*/}}
+{{- define "audit.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "lakefs.name" . }}-audit-maintain
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: audit-maintain
+app: {{ include "lakefs.name" . }}-audit-maintain
+{{- end }}
+
 {{- define "lakefs.dockerConfigJson" }}
 {{- $token := .Values.image.privateRegistry.secretToken }}
 {{- $username := "externallakefs" }}

charts/lakefs/templates/audit/cronjob.yaml

@@ -0,0 +1,109 @@
+{{- if and (.Values.enterprise).enabled (.Values.auditLog).enabled (.Values.auditLog.maintenance).cronJob }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "audit.fullname" . }}
+  labels:
+    {{- include "audit.labels" . | nindent 4 }}
+spec:
+  schedule: {{ .Values.auditLog.maintenance.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: {{ .Values.auditLog.maintenance.successfulJobsHistoryLimit | default 3 }}
+  failedJobsHistoryLimit: {{ .Values.auditLog.maintenance.failedJobsHistoryLimit | default 3 }}
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: {{ .Values.auditLog.maintenance.activeDeadlineSeconds | default 28800 }}
+      template:
+        metadata:
+          labels:
+            {{- include "audit.selectorLabels" . | nindent 12 }}
+          {{- with .Values.auditLog.maintenance.podAnnotations }}
+          annotations:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        spec:
+          {{- if .Values.serviceAccount.create }}
+          serviceAccountName: {{ include "lakefs.serviceAccountName" . }}
+          {{- else if .Values.auditLog.maintenance.serviceAccountName }}
+          serviceAccountName: {{ .Values.auditLog.maintenance.serviceAccountName }}
+          {{- end }}
+          restartPolicy: OnFailure
+          containers:
+            - name: audit-maintain
+              image: "{{ include "lakefs.repository" . }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              args:
+                - audit
+                - maintain
+                - -c
+                - /etc/lakefs/config.yaml
+                {{- if .Values.auditLog.maintenance.retentionDays }}
+                - --retention-days
+                - {{ .Values.auditLog.maintenance.retentionDays | quote }}
+                {{- end }}
+                {{- with .Values.auditLog.maintenance.extraArgs }}
+                {{- toYaml . | nindent 16 }}
+                {{- end }}
+              {{- include "lakefs.env" . | nindent 14 }}
+              volumeMounts:
+                - name: config-volume
+                  mountPath: /etc/lakefs/config.yaml
+                  subPath: config.yaml
+                  readOnly: true
+                {{- if (.Values.enterprise).enabled }}
+                {{- if or (and .Values.secrets .Values.secrets.licenseContents) (and .Values.existingSecret .Values.secretKeys.licenseContentsKey) }}
+                - name: secret-volume-license-token
+                  mountPath: /etc/lakefs/license.tkn
+                  subPath: license.tkn
+                  readOnly: true
+                {{- end }}
+                {{- end }}
+              {{- with .Values.auditLog.maintenance.extraEnvVars }}
+              env:
+                {{- toYaml . | nindent 16 }}
+              {{- end }}
+              {{- with .Values.auditLog.maintenance.resources }}
+              resources:
+                {{- toYaml . | nindent 16 }}
+              {{- end }}
+          volumes:
+            - name: config-volume
+              configMap:
+                name: {{ include "lakefs.fullname" . }}
+                items:
+                  - key: config.yaml
+                    path: config.yaml
+            {{- if (.Values.enterprise).enabled }}
+            {{- if and .Values.existingSecret .Values.secretKeys.licenseContentsKey }}
+            - name: secret-volume-license-token
+              secret:
+                secretName: {{ .Values.existingSecret }}
+                items:
+                  - key: {{ .Values.secretKeys.licenseContentsKey }}
+                    path: license.tkn
+            {{- else if and .Values.secrets .Values.secrets.licenseContents }}
+            - name: secret-volume-license-token
+              secret:
+                secretName: {{ include "lakefs.fullname" . }}
+                items:
+                  - key: license_contents
+                    path: license.tkn
+            {{- end }}
+            {{- end }}
+          {{- if (.Values.image.privateRegistry).enabled }}
+          imagePullSecrets:
+           {{- if (.Values.image.privateRegistry).secretToken }}
+            - name: "docker-registry"
+           {{- else if (.Values.image.privateRegistry).secretName }}
+            - name: {{ .Values.image.privateRegistry.secretName }}
+           {{- end }}
+          {{- end }}
+          {{- with .Values.auditLog.maintenance.nodeSelector }}
+          nodeSelector:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.auditLog.maintenance.tolerations }}
+          tolerations:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+{{- end }}

charts/lakefs/values.yaml

@@ -9,7 +9,7 @@ image:
   community:
     tag: "1.80.0"
   enterprise:
-    tag: "1.81.0"
+    tag: "1.82.0"
   privateRegistry:
     enabled: false
     secretToken: null
@@ -256,5 +256,34 @@ enterprise:
         samlRsaPublicCert: null
         samlRsaPrivateKey: null
 
+# Audit log maintenance CronJob (Enterprise-only).
+# Requires enterprise.enabled: true and audit_log.enabled: true in lakefsConfig.
+# The CronJob uses the same lakeFS config file — no credentials needed.
+auditLog:
+  enabled: false
+  maintenance:
+    # Run maintenance inside the lakeFS server process (default: true).
+    # Disable to rely solely on the external CronJob instead.
+    inProcess: true
+    # Deploy a Kubernetes CronJob for maintenance instead of the default
+    # in-process scheduler. Set to true only if you want maintenance to
+    # run as a separate pod.
+    cronJob: false
+    # Cron schedule for the CronJob (default: every hour).
+    schedule: "0 * * * *"
+    # Snapshot retention in days (0 = no expiration).
+    retentionDays: 90
+    # Maximum time (seconds) the CronJob is allowed to run before K8s kills it (default: 8 hours).
+    activeDeadlineSeconds: 28800
+    # Extra CLI arguments to pass to the maintain command.
+    extraArgs: []
+    extraEnvVars: []
+    podAnnotations: {}
+    resources: {}
+    nodeSelector: {}
+    tolerations: []
+    successfulJobsHistoryLimit: 3
+    failedJobsHistoryLimit: 3
+
 secrets:
   licenseContents: null