Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ DOCKER_USERNAME ?=
IMAGE_TAG ?= latest
TARGETARCH ?= amd64
PLATFORM ?= docker
IMG ?= $(DOCKER_USERNAME)/$(SERVICE_NAME):$(IMAGE_TAG)
GOPROXY ?= https://proxy.golang.org,direct
BAML_RELEASE_BASE_URL ?= https://github.com/boundaryml/baml/releases/download
IMG ?= $(if $(DOCKER_USERNAME),$(DOCKER_USERNAME)/,)$(SERVICE_NAME):$(IMAGE_TAG)

.PHONY: all
all: build
Expand All @@ -28,12 +30,20 @@ run: ## Run DataFlow dev server from host.

.PHONY: docker-build
docker-build: ## Build docker image.
docker buildx build -f core/Dockerfile --platform linux/$(TARGETARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg TARGETARCH=$(TARGETARCH) --build-arg PLATFORM=$(PLATFORM) -t $(IMG) .
docker buildx build -f core/Dockerfile --platform linux/$(TARGETARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg TARGETARCH=$(TARGETARCH) --build-arg PLATFORM=$(PLATFORM) --build-arg GOPROXY=$(GOPROXY) --build-arg BAML_RELEASE_BASE_URL=$(BAML_RELEASE_BASE_URL) -t $(IMG) .

.PHONY: docker-build-no-cache
docker-build-no-cache: ## Build docker image without cache.
docker buildx build --no-cache -f core/Dockerfile --platform linux/$(TARGETARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg TARGETARCH=$(TARGETARCH) --build-arg PLATFORM=$(PLATFORM) --build-arg GOPROXY=$(GOPROXY) --build-arg BAML_RELEASE_BASE_URL=$(BAML_RELEASE_BASE_URL) -t $(IMG) .

.PHONY: docker-push
docker-push: ## Push docker image.
docker push $(IMG)

.PHONY: docker-build-push
docker-build-push: ## Build and push docker image.
docker buildx build -f core/Dockerfile --platform linux/$(TARGETARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg TARGETARCH=$(TARGETARCH) --build-arg PLATFORM=$(PLATFORM) -t $(IMG) --push .
docker buildx build -f core/Dockerfile --platform linux/$(TARGETARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg TARGETARCH=$(TARGETARCH) --build-arg PLATFORM=$(PLATFORM) --build-arg GOPROXY=$(GOPROXY) --build-arg BAML_RELEASE_BASE_URL=$(BAML_RELEASE_BASE_URL) -t $(IMG) --push .

.PHONY: docker-build-push-no-cache
docker-build-push-no-cache: ## Build docker image without cache and push it.
docker buildx build --no-cache -f core/Dockerfile --platform linux/$(TARGETARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg TARGETARCH=$(TARGETARCH) --build-arg PLATFORM=$(PLATFORM) --build-arg GOPROXY=$(GOPROXY) --build-arg BAML_RELEASE_BASE_URL=$(BAML_RELEASE_BASE_URL) -t $(IMG) --push .
7 changes: 5 additions & 2 deletions core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ WORKDIR /app

# Copy core module files
COPY ./core/go.mod ./core/go.sum ./
ARG GOPROXY=https://proxy.golang.org,direct
ENV GOPROXY=${GOPROXY}
RUN go mod download
COPY ./core/ ./

Expand All @@ -55,6 +57,7 @@ RUN if [ -z "${TARGETARCH}" ] || [ -z "${PLATFORM}" ]; then echo "TARGETARCH and
# Pre-download BAML native library for the target architecture (musl for Alpine)
# Extract BAML version from go.mod to stay in sync
ARG TARGETARCH
ARG BAML_RELEASE_BASE_URL=https://github.com/boundaryml/baml/releases/download
RUN if [ -z "$TARGETARCH" ]; then echo "TARGETARCH build arg is required" >&2; exit 1; fi && \
BAML_VERSION=$(grep 'github.com/boundaryml/baml ' go.mod | awk '{print $2}' | sed 's/^v//') && \
if [ "$TARGETARCH" = "amd64" ]; then \
Expand All @@ -65,8 +68,8 @@ RUN if [ -z "$TARGETARCH" ]; then echo "TARGETARCH build arg is required" >&2; e
echo "Unsupported architecture: $TARGETARCH" && exit 1; \
fi && \
mkdir -p /baml-lib && \
curl -L -o /baml-lib/libbaml_cffi.so \
https://github.com/boundaryml/baml/releases/download/${BAML_VERSION}/libbaml_cffi-${BAML_ARCH}-unknown-linux-musl.so && \
curl --fail --location --retry 5 --retry-delay 3 --retry-all-errors -o /baml-lib/libbaml_cffi.so \
"${BAML_RELEASE_BASE_URL}/${BAML_VERSION}/libbaml_cffi-${BAML_ARCH}-unknown-linux-musl.so" && \
echo "Downloaded BAML musl library for ${BAML_ARCH}"

FROM alpine:3.23
Expand Down
16 changes: 10 additions & 6 deletions core/src/sealos/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -443,12 +443,16 @@ func (r *resolver) resolveClickHouseConnectionData(
if port == "" {
port = "9000"
}
return &connectionData{
username: username,
password: password,
host: NormalizeSecretHost(host, namespace),
port: port,
}, nil
if host != "" {
if _, err := strconv.Atoi(port); err == nil {
return &connectionData{
username: username,
password: password,
host: NormalizeSecretHost(host, namespace),
port: port,
}, nil
}
}
}

service, port, err := r.findServiceByPort(ctx, namespace, resourceName, 9000)
Expand Down
51 changes: 51 additions & 0 deletions core/src/sealos/resolver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,57 @@ func TestResolveBootstrapForClickHouseUsesNativeServiceWhenTcpEndpointMissing(t
}
}

func TestResolveBootstrapForClickHouseUsesNativeServiceWhenTcpEndpointIsTemplated(t *testing.T) {
clientset := fake.NewSimpleClientset(
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-house-conn-credential",
Namespace: "ns-admin",
},
Data: map[string][]byte{
"username": []byte("admin"),
"admin-password": []byte("house-password"),
"tcpEndpoint": []byte("test-house-zookeeper:$(SVC_PORT_tcp)"),
},
},
&corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-house-clickhouse",
Namespace: "ns-admin",
Labels: map[string]string{
"app.kubernetes.io/instance": "test-house",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.0.0.2",
Ports: []corev1.ServicePort{
{Name: "http", Port: 8123},
{Name: "tcp", Port: 9000},
},
},
},
)

resolver := &resolver{
kubeconfig: testKubeconfig("ns-admin"),
clientset: clientset,
}

result, err := resolver.ResolveBootstrap(context.Background(), BootstrapInput{
DBType: "clickhouse",
ResourceName: "test-house",
})
if err != nil {
t.Fatalf("expected clickhouse bootstrap to succeed, got %v", err)
}
if result.Host != "test-house-clickhouse.ns-admin.svc" || result.Port != "9000" {
t.Fatalf("expected clickhouse native service endpoint, got %q:%q", result.Host, result.Port)
}
if result.Credentials.Username != "admin" || result.Credentials.Password != "house-password" {
t.Fatalf("expected clickhouse auth from secret, got %#v", result.Credentials)
}
}

func TestResolveBootstrapForClickHouseFallsBackToGenericFields(t *testing.T) {
clientset := fake.NewSimpleClientset(
&corev1.Secret{
Expand Down
23 changes: 18 additions & 5 deletions dataflow/src/components/database/mongodb/ExportCollectionModal.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { createContext, use, useCallback, useState, type ReactNode } from 'react'
import { Download, FileJson, FileText } from 'lucide-react'
import { Download, FileJson, FileSpreadsheet, FileText } from 'lucide-react'
import { useConnectionStore } from '@/stores/useConnectionStore'
import { addAuthHeader } from '@/config/auth-headers'
import { resolveSchemaParam } from '@/utils/database-features'
Expand All @@ -15,13 +15,26 @@ import { useI18n } from '@/i18n/useI18n'
// Constants
// ---------------------------------------------------------------------------

type CollectionExportFormat = 'json' | 'csv'
type CollectionExportFormat = 'json' | 'csv' | 'excel'

const FORMAT_OPTIONS: FormatOption<CollectionExportFormat>[] = [
{ id: 'json', label: 'JSON', icon: FileJson },
{ id: 'csv', label: 'CSV', icon: FileText },
{ id: 'excel', label: 'Excel', icon: FileSpreadsheet },
]

const BACKEND_FORMATS: Record<CollectionExportFormat, 'ndjson' | 'csv' | 'excel'> = {
json: 'ndjson',
csv: 'csv',
excel: 'excel',
}

const FORMAT_EXTENSIONS: Record<CollectionExportFormat, string> = {
json: 'ndjson',
csv: 'csv',
excel: 'xlsx',
}

// ---------------------------------------------------------------------------
// Context
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -113,7 +126,7 @@ function ExportCollectionBridge({
if (!connection) throw new Error(t('common.error.connectionNotFound'))

const graphqlSchema = resolveSchemaParam(connection.type, databaseName)
const backendFormat = format === 'json' ? 'ndjson' : 'csv'
const backendFormat = BACKEND_FORMATS[format]

const response = await fetch('/api/export', {
method: 'POST',
Expand All @@ -138,7 +151,7 @@ function ExportCollectionBridge({
const disposition = response.headers.get('Content-Disposition')
const filenameMatch = disposition?.match(/filename="(.+)"/)
const filename =
filenameMatch?.[1] ?? `${collectionName}_export.${format === 'json' ? 'ndjson' : 'csv'}`
filenameMatch?.[1] ?? `${collectionName}_export.${FORMAT_EXTENSIONS[format]}`

const blob = await response.blob()
downloadBlob(blob, filename)
Expand All @@ -153,7 +166,7 @@ function ExportCollectionBridge({
} finally {
actions.setSubmitting(false)
}
}, [actions, collectionName, connectionId, connections, databaseName, format, t])
}, [actions, collectionName, connectionId, connections, databaseName, filter, format, limit, t])

return (
<ExportCollectionCtx
Expand Down
122 changes: 122 additions & 0 deletions dataflow/src/test/ExportCollectionModal.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import { fireEvent, screen, waitFor } from '@testing-library/react'
import { beforeEach, describe, expect, it, vi } from 'vitest'

import { ExportCollectionModal } from '@/components/database/mongodb/ExportCollectionModal'
import { downloadBlob } from '@/utils/export-utils'
import { renderWithI18n } from '@/test/renderWithI18n'
import { useConnectionStore, type Connection } from '@/stores/useConnectionStore'

vi.mock('@/utils/export-utils', async (importOriginal) => ({
...(await importOriginal<typeof import('@/utils/export-utils')>()),
downloadBlob: vi.fn(),
}))

const originalState = useConnectionStore.getState()

const mongoConnection: Connection = {
id: 'mongo-1',
name: 'MongoDB @ localhost',
type: 'MONGODB',
host: 'localhost',
port: '27017',
user: 'root',
password: '',
database: 'admin',
createdAt: '2026-04-02T00:00:00.000Z',
}

describe('ExportCollectionModal', () => {
beforeEach(() => {
useConnectionStore.setState(originalState)
vi.mocked(downloadBlob).mockReset()
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(
new Response(new Blob(['xlsx']), {
status: 200,
headers: {
'Content-Disposition': 'attachment; filename="analytics_events.xlsx"',
},
}),
),
)
})

it('exports a MongoDB collection as Excel through the export endpoint', async () => {
useConnectionStore.setState({
...useConnectionStore.getState(),
connections: [mongoConnection],
})

renderWithI18n(
<ExportCollectionModal
open
onOpenChange={vi.fn()}
connectionId={mongoConnection.id}
databaseName="analytics"
collectionName="events"
/>,
'en',
)

fireEvent.click(screen.getByRole('button', { name: 'Excel' }))
fireEvent.click(screen.getByRole('button', { name: 'Start Export' }))

await waitFor(() => {
expect(globalThis.fetch).toHaveBeenCalledWith(
'/api/export',
expect.objectContaining({
method: 'POST',
body: JSON.stringify({
schema: 'analytics',
storageUnit: 'events',
format: 'excel',
}),
}),
)
})

expect(downloadBlob).toHaveBeenCalledWith(expect.any(Blob), 'analytics_events.xlsx')
})

it('includes the latest filter and limit when exporting', async () => {
useConnectionStore.setState({
...useConnectionStore.getState(),
connections: [mongoConnection],
})

renderWithI18n(
<ExportCollectionModal
open
onOpenChange={vi.fn()}
connectionId={mongoConnection.id}
databaseName="analytics"
collectionName="events"
/>,
'en',
)

fireEvent.change(screen.getByPlaceholderText('{ "status": "active" }'), {
target: { value: '{ "status": "active" }' },
})
fireEvent.change(screen.getByPlaceholderText('No limit'), {
target: { value: '25' },
})
fireEvent.click(screen.getByRole('button', { name: 'Start Export' }))

await waitFor(() => {
expect(globalThis.fetch).toHaveBeenCalledWith(
'/api/export',
expect.objectContaining({
body: JSON.stringify({
schema: 'analytics',
storageUnit: 'events',
format: 'ndjson',
filter: '{ "status": "active" }',
limit: 25,
}),
}),
)
})
})
})
Loading