Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# SENTRY_TRACES_SAMPLE_RATE=0
# VITE_SENTRY_TRACES_SAMPLE_RATE=0

DATABASE_URL="file:../data/data.db"
UPFLOW_DATA_DIR=./data
BETTER_AUTH_SECRET=your-better-auth-secret-at-least-32-chars
GITHUB_CLIENT_ID=your-github-client-id
GITHUB_CLIENT_SECRET=your-github-client-secret
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ jobs:

- name: 🛠 Setup Database
run: pnpm db:setup
env:
UPFLOW_DATA_DIR: ./data

- name: 🔎 Check generated types are up to date
run: |
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,6 @@ opensrc/

# Sentry Config File
.env.sentry-build-plugin
data.db
data.db-shm
data.db-wal
3 changes: 1 addition & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,12 @@ RUN SENTRY_DSN="$SENTRY_DSN" \
# --- Production image ---
FROM runtime-base

ENV DATABASE_URL="file:/upflow/data/data.db?connection_limit=1"
ENV UPFLOW_DATA_DIR="/upflow/data"
ENV PORT="8080"
ENV NODE_ENV="production"

# add shortcut for connecting to database CLI
RUN printf '#!/bin/sh\nset -x\nsqlite3 file:/upflow/data/data.db\n' > /usr/local/bin/database-cli && chmod +x /usr/local/bin/database-cli
RUN printf '#!/bin/sh\nset -x\nsqlite3 "${UPFLOW_DATA_DIR}/data.db"\n' > /usr/local/bin/database-cli && chmod +x /usr/local/bin/database-cli

WORKDIR /upflow

Expand Down
18 changes: 9 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,15 @@ cp .env.example .env

Edit `.env` with your values:

| Variable | Description | Required |
| --------------------------- | -------------------------------------------------- | -------- |
| `DATABASE_URL` | SQLite database path (e.g. `file:../data/data.db`) | Yes |
| `BETTER_AUTH_SECRET` | Secret for better-auth (min 32 chars) | Yes |
| `BETTER_AUTH_URL` | App URL (e.g. `http://localhost:5173`) | Yes |
| `GITHUB_CLIENT_ID` | GitHub App client ID | Yes |
| `GITHUB_CLIENT_SECRET` | GitHub App client secret | Yes |
| `INTEGRATION_PRIVATE_TOKEN` | GitHub PAT for PR data fetching | Yes |
| `GEMINI_API_KEY` | Gemini API key for PR classification | No |
| Variable | Description | Required |
| --------------------------- | -------------------------------------- | -------- |
| `UPFLOW_DATA_DIR` | Data directory path (e.g. `./data`) | Yes |
| `BETTER_AUTH_SECRET` | Secret for better-auth (min 32 chars) | Yes |
| `BETTER_AUTH_URL` | App URL (e.g. `http://localhost:5173`) | Yes |
| `GITHUB_CLIENT_ID` | GitHub App client ID | Yes |
| `GITHUB_CLIENT_SECRET` | GitHub App client secret | Yes |
| `INTEGRATION_PRIVATE_TOKEN` | GitHub PAT for PR data fetching | Yes |
| `GEMINI_API_KEY` | Gemini API key for PR classification | No |

### 3. Set up GitHub App

Expand Down
11 changes: 9 additions & 2 deletions app/libs/auth.server.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
import { describe, expect, test } from 'vitest'
import { safeRedirectTo } from './auth.server'
import { mkdtempSync } from 'node:fs'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { describe, expect, test, vi } from 'vitest'

vi.stubEnv('UPFLOW_DATA_DIR', mkdtempSync(path.join(tmpdir(), 'auth-test-')))

// Import after env stub to avoid resolveDataDir() throwing
const { safeRedirectTo } = await import('./auth.server')

describe('safeRedirectTo', () => {
test('returns the path when it starts with /', () => {
Expand Down
2 changes: 1 addition & 1 deletion app/libs/dotenv.server.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { z } from 'zod'

const envSchema = z.object({
DATABASE_URL: z.string(),
UPFLOW_DATA_DIR: z.string(),
BETTER_AUTH_URL: z.string(),
BETTER_AUTH_SECRET: z.string().min(32),
GOOGLE_CLIENT_ID: z.string(),
Expand Down
3 changes: 1 addition & 2 deletions app/libs/github-app-state.server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@ rawInit.exec(`
`)
rawInit.close()

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

describe('github-app-state', () => {
const orgId = toOrgId('org-1')
Expand Down
3 changes: 1 addition & 2 deletions app/libs/timezone.server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '')

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

function createTenantWithSettings(orgId: string, timezone?: string) {
const dbPath = path.join(testDir, `tenant_${orgId}.db`)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,7 @@ sharedDb.exec(`
`)
sharedDb.close()

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${sharedDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(sharedDbPath))

let testCounter = 0
function createFreshOrg(): OrganizationId {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '')

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

let testCounter = 0
function createFreshOrg(): {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '')

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

let testCounter = 0
function createFreshOrg(): {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '')

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

let testCounter = 0
function createFreshOrg(): OrganizationId {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '')

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

let testCounter = 0
function createFreshOrg(): OrganizationId {
Expand Down
9 changes: 5 additions & 4 deletions app/services/db.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,12 @@ import {
type Selectable,
type Updateable,
} from 'kysely'
import path from 'node:path'
import { resolveDataDir } from '~/db/data-dir'
import type * as DB from './type'

export { resolveDataDir }

const debug = createDebug('app:db')
const SQLITE_BUSY_TIMEOUT_MS = 5000

Expand All @@ -29,10 +33,7 @@ let sharedDbState: SharedDbState | undefined
function getSharedDbState(): SharedDbState {
if (sharedDbState) return sharedDbState

if (!process.env.DATABASE_URL) {
throw new Error('DATABASE_URL environment variable is required')
}
const filename = `${process.env.NODE_ENV === 'production' ? '' : '.'}${new URL(process.env.DATABASE_URL).pathname}`
const filename = path.join(resolveDataDir(), 'data.db')
const database = new SQLite(filename)
database.pragma('journal_mode = WAL')
database.pragma(`busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`)
Expand Down
3 changes: 1 addition & 2 deletions app/services/github-webhook.server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,7 @@ rawInit.exec(`
`)
rawInit.close()

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

describe('processGithubWebhookPayload', () => {
const clearSpy = vi.spyOn(cache, 'clearOrgCache')
Expand Down
6 changes: 2 additions & 4 deletions app/services/tenant-db.server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,8 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '') // create empty shared DB file

// getTenantDbPath uses: `${NODE_ENV === 'production' ? '' : '.'}${new URL(DATABASE_URL).pathname}`
// In production mode, pathname is used as-is. Use production mode for predictable paths.
vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
// getTenantDbPath uses: path.join(resolveDataDir(), `tenant_${orgId}.db`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

function createTestTenantDb(orgId: string): string {
const dbPath = path.join(testDir, `tenant_${orgId}.db`)
Expand Down
8 changes: 2 additions & 6 deletions app/services/tenant-db.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { execFileSync } from 'node:child_process'
import { existsSync, unlinkSync } from 'node:fs'
import path from 'node:path'
import type { OrganizationId } from '~/app/types/organization'
import { resolveDataDir } from './db.server'
import type * as TenantDB from './tenant-type'

export type { TenantDB }
Expand All @@ -26,12 +27,7 @@ const tenantDbCache = new Map<
>()

function getTenantDbPath(organizationId: OrganizationId): string {
if (!process.env.DATABASE_URL) {
throw new Error('DATABASE_URL environment variable is required')
}
const sharedDbPath = `${process.env.NODE_ENV === 'production' ? '' : '.'}${new URL(process.env.DATABASE_URL).pathname}`
const dir = path.dirname(sharedDbPath)
return path.join(dir, `tenant_${organizationId}.db`)
return path.join(resolveDataDir(), `tenant_${organizationId}.db`)
}

function ensureTenantDb(organizationId: OrganizationId) {
Expand Down
3 changes: 1 addition & 2 deletions batch/db/mutations.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '')

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

const orgId = toOrgId(`test-org-${Date.now()}`)

Expand Down
3 changes: 1 addition & 2 deletions batch/github/store.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ mkdirSync(testDir, { recursive: true })
const testDbPath = path.join(testDir, 'data.db')
writeFileSync(testDbPath, '')

vi.stubEnv('NODE_ENV', 'production')
vi.stubEnv('DATABASE_URL', `file://${testDbPath}`)
vi.stubEnv('UPFLOW_DATA_DIR', path.dirname(testDbPath))

afterAll(() => {
vi.unstubAllEnvs()
Expand Down
3 changes: 2 additions & 1 deletion db/apply-tenant-migrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import { consola } from 'consola'
import { execFileSync } from 'node:child_process'
import { readdirSync } from 'node:fs'
import { join } from 'node:path'
import { resolveDataDir } from './data-dir'

const dataDir = './data'
const dataDir = resolveDataDir()
const migrationsDir = './db/migrations/tenant'

function getTenantDbFiles(): string[] {
Expand Down
8 changes: 8 additions & 0 deletions db/data-dir.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/** Resolve data directory from UPFLOW_DATA_DIR env var. Dependency-free. */
export function resolveDataDir(): string {
const dir = process.env.UPFLOW_DATA_DIR
if (!dir) {
throw new Error('UPFLOW_DATA_DIR environment variable is required')
}
return dir
}
20 changes: 5 additions & 15 deletions db/migrate-integrations-to-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,9 @@ import { consola } from 'consola'
import 'dotenv/config'
import { existsSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
import { resolveDataDir } from './data-dir'

const dataDir = join(process.cwd(), 'data')

function sharedDbFilePath(): string {
const url = process.env.DATABASE_URL ?? 'file:./data/data.db'
const pathname = new URL(url).pathname
const normalized =
process.env.NODE_ENV === 'production'
? pathname
: `.${pathname.replace(/^\./, '')}`
return join(process.cwd(), normalized)
}
const dataDir = resolveDataDir()

function hasTable(db: InstanceType<typeof Database>, name: string): boolean {
const row = db
Expand All @@ -30,12 +21,11 @@ function hasTable(db: InstanceType<typeof Database>, name: string): boolean {
}

function migrate(): void {
const sharedPath = sharedDbFilePath()
const sharedPath = join(dataDir, 'data.db')
if (!existsSync(sharedPath)) {
consola.info(
`Shared DB missing at ${sharedPath}; skipping integrations migration.`,
throw new Error(
`Shared DB not found at ${sharedPath}. Check UPFLOW_DATA_DIR (current: ${process.env.UPFLOW_DATA_DIR ?? '(unset)'}).`,
)
return
}

const shared = new Database(sharedPath)
Expand Down
4 changes: 1 addition & 3 deletions start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@

set -ex

DB_URL='sqlite:///upflow/data/data.db'

# 1. Apply shared DB migrations
atlas migrate apply --env local --url "$DB_URL"
atlas migrate apply --env local --url "sqlite://${UPFLOW_DATA_DIR}/data.db"

# 2. Migrate integrations data from tenant DBs to shared DB (idempotent, safe to re-run)
node build/db/migrate-integrations-to-shared.js
Expand Down