feat: fs support with drizzle and duckdb-wasm (#30)

* feat: fs support with drizzle and duckdb-wasm

* docs: fix lint errors

* fix: apply suggestions

* fix: lockfile
This commit is contained in:
Makito
2025-02-22 21:44:13 +08:00
committed by Neko Ayaka
parent 9011d3e4ac
commit aeb17c7a7c
18 changed files with 1429 additions and 152 deletions
+6 -2
View File
@@ -55,7 +55,9 @@
"play:build": "vite build",
"play:preview": "vite preview",
"typecheck": "tsc --noEmit",
"db:generate": "drizzle-kit generate"
"db:generate": "drizzle-kit generate",
"test": "vitest",
"test:run": "vitest run"
},
"peerDependencies": {
"web-worker": "^1.5.0"
@@ -67,7 +69,7 @@
},
"dependencies": {
"@date-fns/tz": "^1.2.0",
"@duckdb/duckdb-wasm": "^1.29.0",
"@duckdb/duckdb-wasm": "1.29.1-dev68.0",
"@proj-airi/duckdb-wasm": "workspace:^",
"apache-arrow": "^19.0.1",
"date-fns": "^4.1.0",
@@ -78,8 +80,10 @@
"devDependencies": {
"@unocss/reset": "^66.0.0",
"@vitejs/plugin-vue": "^5.2.1",
"@vitest/browser": "^3.0.6",
"@vueuse/core": "^12.7.0",
"drizzle-kit": "^0.30.4",
"playwright": "^1.50.1",
"superjson": "^2.2.2",
"vite": "^6.1.1",
"vue": "^3.5.13",
@@ -1,11 +1,13 @@
<script setup lang="ts">
import type { DuckDBWasmDrizzleDatabase } from '../../src'
import { useDebounceFn } from '@vueuse/core'
import { DuckDBAccessMode } from '@duckdb/duckdb-wasm'
import { DBStorageType } from '@proj-airi/duckdb-wasm'
import { serialize } from 'superjson'
import { onMounted, onUnmounted, ref, watch } from 'vue'
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { drizzle } from '../../src'
import { buildDSN } from '../../src/dsn'
import * as schema from '../db/schema'
import { users } from '../db/schema'
import migration1 from '../drizzle/0000_cute_kulan_gath.sql?raw'
@@ -13,18 +15,40 @@ import migration1 from '../drizzle/0000_cute_kulan_gath.sql?raw'
const db = ref<DuckDBWasmDrizzleDatabase<typeof schema>>()
const results = ref<Record<string, unknown>[]>()
const schemaResults = ref<Record<string, unknown>[]>()
const query = ref(`SELECT 1 + 1 AS result`)
const isMigrated = ref(false)
onMounted(async () => {
db.value = drizzle('duckdb-wasm://?bundles=import-url', { schema })
const storage = ref<DBStorageType>()
const path = ref('test.db')
const logger = ref(true)
const readOnly = ref(false)
const dsn = computed(() => {
return buildDSN({
scheme: 'duckdb-wasm:',
bundles: 'import-url',
logger: logger.value,
...storage.value === DBStorageType.ORIGIN_PRIVATE_FS && {
storage: {
type: storage.value,
path: path.value,
accessMode: readOnly.value ? DuckDBAccessMode.READ_ONLY : DuckDBAccessMode.READ_WRITE,
},
},
})
})
const query = ref(`SELECT * FROM 'users'`)
async function connect() {
isMigrated.value = false
db.value = drizzle(dsn.value, { schema })
await db.value?.execute('INSTALL vss;')
await db.value?.execute('LOAD vss;')
}
async function migrate() {
await db.value?.execute(migration1)
results.value = await db.value?.execute(query.value)
await db.value.insert(users).values({
await db.value?.insert(users).values({
id: '9449af72-faad-4c97-8a45-69f9f1ca1b05',
decimal: '1.23456',
numeric: '1.23456',
@@ -32,58 +56,203 @@ onMounted(async () => {
double: 1.23456,
interval: '365 day',
})
isMigrated.value = true
}
const usersResults = await db.value.select().from(users)
async function insert() {
await db.value?.insert(users).values({
id: crypto.randomUUID().replace(/-/g, ''),
decimal: '1.23456',
numeric: '1.23456',
real: 1.23456,
double: 1.23456,
interval: '365 day',
})
}
async function reconnect() {
const client = await db.value?.$client
await client?.close()
await connect()
}
async function execute() {
results.value = await db.value?.execute(query.value)
}
async function executeORM() {
schemaResults.value = await db.value?.select().from(users)
}
async function shallowListOPFS() {
const opfsRoot = await navigator.storage.getDirectory()
const files: string[] = []
for await (const name of opfsRoot.keys()) {
files.push(name)
}
// eslint-disable-next-line no-console
console.log(['Files in OPFS:', ...files].join('\n'))
}
async function wipeOPFS() {
await db.value?.$client.then(client => client.close())
const opfsRoot = await navigator.storage.getDirectory()
const promises: Promise<void>[] = []
for await (const name of opfsRoot.keys()) {
promises.push(opfsRoot.removeEntry(name, { recursive: true }).then(() => {
// eslint-disable-next-line no-console
console.info(`File removed from OPFS: "${name}"`)
}))
}
await Promise.all(promises)
}
onMounted(async () => {
await connect()
await migrate()
results.value = await db.value?.execute(query.value)
const usersResults = await db.value?.select().from(users)
schemaResults.value = usersResults
})
onUnmounted(() => {
db.value?.$client.then(client => client.close())
})
watch(query, useDebounceFn(async () => {
results.value = await db.value?.execute(query.value)
}, 1000))
</script>
<template>
<div flex flex-col gap-4 p-4>
<div flex flex-col gap-2 p-4>
<h1 text-2xl>
<code>@duckdb/duckdb-wasm</code> + <code>drizzle-orm</code> Playground
</h1>
<div flex flex-col gap-2>
<h2 text-xl>
Executing
Storage
</h2>
<div>
<textarea v-model="query" h-full w-full rounded-lg bg="neutral-100 dark:neutral-800" p-4 font-mono />
<div flex flex-row gap-2>
<div flex flex-row gap-2>
<input id="in-memory" v-model="storage" type="radio" :value="undefined">
<label for="in-memory">In-Memory</label>
</div>
<div flex flex-row gap-2>
<input id="opfs" v-model="storage" type="radio" :value="DBStorageType.ORIGIN_PRIVATE_FS">
<label for="opfs">Origin Private FS</label>
</div>
</div>
</div>
<div grid grid-cols-3 gap-2>
<div flex flex-col gap-2>
<h2 text-xl>
Logger
</h2>
<div flex flex-row gap-2>
<input id="logger" v-model="logger" type="checkbox">
<label for="logger">Enable</label>
</div>
</div>
<div flex flex-col gap-2>
<h2 text-xl>
Read-only
</h2>
<div flex flex-row gap-2>
<input id="readOnly" v-model="readOnly" type="checkbox">
<label for="readOnly">Read-only (DB file creation will fail)</label>
</div>
</div>
</div>
<div v-if="storage === DBStorageType.ORIGIN_PRIVATE_FS" flex flex-col gap-2>
<h2 text-xl>
Path
</h2>
<div flex flex-col gap-1>
<input v-model="path" type="text" w-full rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
<div text-sm>
<ul list-disc-inside>
<li>
Leading slash is optional ("/path/to/database.db" is equivalent to "path/to/database.db")
</li>
<li>Empty path is INVALID</li>
</ul>
</div>
</div>
</div>
<div flex flex-col gap-2>
<h2 text-xl>
Results
</h2>
<div whitespace-pre-wrap p-4 font-mono>
{{ JSON.stringify(serialize(results).json, null, 2) }}
</div>
</div>
<div flex flex-col gap-2>
<h2 text-xl>
Executing
DSN (read-only)
</h2>
<div>
<pre whitespace-pre-wrap rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
<input v-model="dsn" readonly type="text" w-full rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
</div>
</div>
<div flex flex-row justify-between gap-2>
<div flex flex-row gap-2>
<button rounded-lg bg="pink-100 dark:pink-700" px-4 py-2 @click="reconnect">
Reconnect
</button>
<button rounded-lg bg="orange-100 dark:orange-700" px-4 py-2 :class="{ 'cursor-not-allowed': isMigrated }" :disabled="isMigrated" @click="migrate">
{{ isMigrated ? 'Already migrated 🥳' : 'Migrate' }}
</button>
<button rounded-lg bg="purple-100 dark:purple-700" px-4 py-2 @click="insert">
Insert
</button>
</div>
<div flex flex-row gap-2>
<button rounded-lg bg="green-100 dark:green-700" px-4 py-2 @click="shallowListOPFS">
List OPFS (See console)
</button>
<button rounded-lg bg="red-100 dark:red-700" px-4 py-2 @click="wipeOPFS">
Wipe OPFS
</button>
</div>
</div>
<div grid grid-cols-2 gap-2>
<div flex flex-col gap-2>
<h2 text-xl>
Executing
</h2>
<div>
<textarea v-model="query" h-full w-full rounded-lg bg="neutral-100 dark:neutral-800" p-4 font-mono />
</div>
<div flex flex-row gap-2>
<button rounded-lg bg="blue-100 dark:blue-700" px-4 py-2 @click="execute">
Execute
</button>
</div>
<div flex flex-col gap-2>
<h2 text-xl>
Results
</h2>
<div whitespace-pre-wrap p-4 font-mono>
{{ JSON.stringify(serialize(results).json, null, 2) }}
</div>
</div>
</div>
<div>
<div flex flex-col gap-2>
<h2 text-xl>
Executing (ORM, read-only)
</h2>
<div>
<pre whitespace-pre-wrap rounded-lg p-4 font-mono bg="neutral-100 dark:neutral-800">
await db.insert(users).values({ id: '9449af72-faad-4c97-8a45-69f9f1ca1b05' })
await db.select().from(users)
</pre>
</div>
</div>
<div flex flex-col gap-2>
<h2 text-xl>
Schema Results
</h2>
<div whitespace-pre-wrap p-4 font-mono>
{{ JSON.stringify(serialize(schemaResults).json, null, 2) }}
</pre>
</div>
<div flex flex-row gap-2>
<button rounded-lg bg="blue-100 dark:blue-700" px-4 py-2 @click="executeORM">
Execute
</button>
</div>
</div>
<div flex flex-col gap-2>
<h2 text-xl>
Schema Results
</h2>
<div whitespace-pre-wrap p-4 font-mono>
{{ JSON.stringify(serialize(schemaResults).json, null, 2) }}
</div>
</div>
</div>
</div>
</div>
@@ -0,0 +1,223 @@
import { DuckDBAccessMode } from '@duckdb/duckdb-wasm'
import { DBStorageType } from '@proj-airi/duckdb-wasm'
import { afterAll, beforeAll, describe, expect, it, onTestFinished } from 'vitest'
import { drizzle } from '.'
import { getImportUrlBundles } from './bundles/import-url-browser'
describe('drizzle with duckdb wasm in browser', { timeout: 10000 }, async () => {
beforeAll(async () => {
const opfsRoot = await navigator.storage.getDirectory()
for await (const name of opfsRoot.keys()) {
if (name.startsWith('drizzle_test_')) {
await opfsRoot.removeEntry(name)
}
}
})
afterAll(async () => {
const opfsRoot = await navigator.storage.getDirectory()
for await (const name of opfsRoot.keys()) {
if (name.startsWith('drizzle_test_')) {
await opfsRoot.removeEntry(name)
}
}
})
it('should have navigator.storage.getDirectory', async () => {
const getDirectory = navigator.storage?.getDirectory
expect(typeof getDirectory).toBe('function')
})
it('should connect to an in-memory DuckDB WASM database', async () => {
const db = drizzle({ connection: { bundles: getImportUrlBundles() } })
const res = await db.execute('SELECT count(*)::INTEGER as v FROM generate_series(0, 100) t(v)')
expect(res).toBeDefined()
expect(res).toEqual([{ v: 101 }])
})
// TODO: Enable this test when DuckDB no longer creates files in read-only mode
it.skip('should fail to open a non-existent OPFS database', async () => {
const db = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: 'drizzle_test_non_existent',
accessMode: DuckDBAccessMode.READ_ONLY,
},
},
})
// No need to close as the DB will fail to open
await expect(db.$client).rejects.toThrow(/file or directory could not be found/)
const opfsRoot = await navigator.storage.getDirectory()
const nonExistentFileHandle = opfsRoot.getFileHandle('drizzle_test_non_existent', { create: false })
await expect(nonExistentFileHandle).rejects.toThrow(/file or directory could not be found/)
})
it('should create and open an OPFS database', async () => {
const path = `drizzle_test_${crypto.randomUUID().replace(/-/g, '')}`
const db = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: `${path}`,
accessMode: DuckDBAccessMode.READ_WRITE,
},
},
})
onTestFinished(async () => (await db.$client).close())
await expect(db.$client).resolves.toBeDefined()
})
it('should not create an OPFS database with an empty path', async () => {
const db = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: '',
accessMode: DuckDBAccessMode.READ_WRITE,
},
},
})
// No need to close as the DB will fail to open
await expect(db.$client).rejects.toThrow(/Name is not allowed/)
})
it('should not create an OPFS database with an invalid path', async () => {
const path = `//drizzle_test_${crypto.randomUUID().replace(/-/g, '')}`
const db = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: `${path}`,
accessMode: DuckDBAccessMode.READ_WRITE,
},
},
})
// No need to close as the DB will fail to open
await expect(db.$client).rejects.toThrow(/Name is not allowed/)
})
it('should open, update, save, and reload an OPFS database', async () => {
const path = `drizzle_test_${crypto.randomUUID().replace(/-/g, '')}`
const db1 = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: `${path}`,
accessMode: DuckDBAccessMode.READ_WRITE,
},
},
})
onTestFinished(async () => (await db1.$client).close())
await expect(db1.$client).resolves.toBeDefined()
expect(await db1.execute('SHOW TABLES')).toEqual([])
await expect(db1.execute('CREATE TABLE test (v INTEGER)')).resolves.toBeDefined()
await expect(db1.execute('INSERT INTO test VALUES (1), (2), (3)')).resolves.toBeDefined()
expect(await db1.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
await expect(db1.execute('CHECKPOINT')).resolves.toBeDefined()
await expect((await db1.$client).close()).resolves.toBeUndefined()
const db2 = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: `${path}`,
accessMode: DuckDBAccessMode.READ_ONLY,
},
},
})
onTestFinished(async () => (await db2.$client).close())
expect(await db2.execute('SHOW TABLES')).toEqual([{ name: 'test' }])
expect(await db2.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
})
it('should create open the same OPFS database with or without a leading slash', async () => {
const path = `drizzle_test_${crypto.randomUUID().replace(/-/g, '')}`
const db1 = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: `${path}`,
accessMode: DuckDBAccessMode.READ_WRITE,
},
},
})
onTestFinished(async () => (await db1.$client).close())
await expect(db1.$client).resolves.toBeDefined()
expect(await db1.execute('SHOW TABLES')).toEqual([])
await expect(db1.execute('CREATE TABLE test (v INTEGER)')).resolves.toBeDefined()
await expect(db1.execute('INSERT INTO test VALUES (1), (2), (3)')).resolves.toBeDefined()
expect(await db1.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
await expect(db1.execute('CHECKPOINT')).resolves.toBeDefined()
await expect((await db1.$client).close()).resolves.toBeUndefined()
const db2 = drizzle({
connection: {
bundles: getImportUrlBundles(),
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: `/${path}`,
accessMode: DuckDBAccessMode.READ_ONLY,
},
},
})
onTestFinished(async () => (await db2.$client).close())
expect(await db2.execute('SHOW TABLES')).toEqual([{ name: 'test' }])
expect(await db2.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
})
it('should create, open, update, save, and reload an OPFS database with DSN', async () => {
const path = `drizzle_test_${crypto.randomUUID().replace(/-/g, '')}`
const dsn = `duckdb-wasm:///${path}?bundles=import-url&storage=origin-private-fs&write=true`
const db1 = drizzle(dsn)
onTestFinished(async () => (await db1.$client).close())
await expect(db1.$client).resolves.toBeDefined()
expect(await db1.execute('SHOW TABLES')).toEqual([])
await expect(db1.execute('CREATE TABLE test (v INTEGER)')).resolves.toBeDefined()
await expect(db1.execute('INSERT INTO test VALUES (1), (2), (3)')).resolves.toBeDefined()
expect(await db1.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
await expect(db1.execute('CHECKPOINT')).resolves.toBeDefined()
await expect((await db1.$client).close()).resolves.toBeUndefined()
const db2 = drizzle(dsn)
onTestFinished(async () => (await db2.$client).close())
expect(await db2.execute('SHOW TABLES')).toEqual([{ name: 'test' }])
expect(await db2.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
})
})
@@ -0,0 +1,107 @@
import type { DBNodeFS } from '@proj-airi/duckdb-wasm'
import { randomUUID } from 'node:crypto'
import { readdir, unlink } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { DuckDBAccessMode } from '@duckdb/duckdb-wasm'
import { DBStorageType } from '@proj-airi/duckdb-wasm'
import { getBundles } from '@proj-airi/duckdb-wasm/bundles/default-node'
import { afterAll, beforeAll, describe, expect, it, onTestFinished } from 'vitest'
import { drizzle } from '.'
describe('drizzle with duckdb wasm in node', { timeout: 10000 }, async () => {
beforeAll(async () => {
const tmp = tmpdir()
await Promise.all(
(await readdir(tmp)).reduce<Promise<void>[]>((tasks, filename) => {
if (filename.startsWith('drizzle_test_')) {
tasks.push(unlink(path.join(tmp, filename)))
}
return tasks
}, []),
)
})
afterAll(async () => {
const tmp = tmpdir()
await Promise.all(
(await readdir(tmp)).reduce<Promise<void>[]>((tasks, filename) => {
if (filename.startsWith('drizzle_test_')) {
tasks.push(unlink(path.join(tmp, filename)))
}
return tasks
}, []),
)
})
it('should connect to an in-memory DuckDB WASM database', async () => {
const db = drizzle({ connection: { bundles: getBundles() } })
const res = await db.execute('SELECT count(*)::INTEGER as v FROM generate_series(0, 100) t(v)')
expect(res).toBeDefined()
expect(res).toEqual([{ v: 101 }])
})
it('should open a DuckDB WASM database in Node FS', async () => {
const tmp = tmpdir()
const filename = `drizzle_test_${randomUUID().replace(/-/g, '')}`
const db = drizzle({
connection: {
bundles: getBundles(),
storage: {
type: DBStorageType.NODE_FS,
path: path.resolve(tmp, filename),
accessMode: DuckDBAccessMode.READ_WRITE,
} as DBNodeFS,
},
})
await expect(db.$client).resolves.toBeDefined()
})
it('should open, update, save, and reload an OPFS database', async () => {
const tmp = tmpdir()
const filename = `drizzle_test_${randomUUID().replace(/-/g, '')}`
const db1 = drizzle({
connection: {
bundles: getBundles(),
storage: {
type: DBStorageType.NODE_FS,
path: path.resolve(tmp, filename),
accessMode: DuckDBAccessMode.READ_WRITE,
},
},
})
onTestFinished(async () => (await db1.$client).close())
await expect(db1.$client).resolves.toBeDefined()
expect(await db1.execute('SHOW TABLES')).toEqual([])
await expect(db1.execute('CREATE TABLE test (v INTEGER)')).resolves.toBeDefined()
await expect(db1.execute('INSERT INTO test VALUES (1), (2), (3)')).resolves.toBeDefined()
expect(await db1.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
await expect(db1.execute('CHECKPOINT')).resolves.toBeDefined()
await expect((await db1.$client).close()).resolves.toBeUndefined()
const db2 = drizzle({
connection: {
bundles: getBundles(),
storage: {
type: DBStorageType.NODE_FS,
path: path.resolve(tmp, filename),
accessMode: DuckDBAccessMode.READ_ONLY,
},
},
})
onTestFinished(async () => (await db2.$client).close())
expect(await db2.execute('SHOW TABLES')).toEqual([{ name: 'test' }])
expect(await db2.execute('SELECT * FROM test')).toEqual([{ v: 1 }, { v: 2 }, { v: 3 }])
})
})
+45 -65
View File
@@ -1,5 +1,5 @@
import type { DuckDBBundles, Logger } from '@duckdb/duckdb-wasm'
import type { DuckDBWasmClient } from '@proj-airi/duckdb-wasm'
import type { DuckDBBundles } from '@duckdb/duckdb-wasm'
import type { ConnectOptions, DuckDBWasmClient } from '@proj-airi/duckdb-wasm'
import type { DrizzleConfig, RelationalSchemaConfig, TablesRelationalConfig } from 'drizzle-orm'
import type { DuckDBWasmQueryResultHKT } from './session'
@@ -8,6 +8,7 @@ import { connect, getEnvironment } from '@proj-airi/duckdb-wasm'
import { createTableRelationsHelpers, DefaultLogger, entityKind, extractTablesRelationalConfig, isConfig } from 'drizzle-orm'
import { PgDatabase, PgDialect } from 'drizzle-orm/pg-core'
import { parseDSN } from './dsn'
import { DuckDBWasmSession } from './session'
export class DuckDBWasmDatabase<
@@ -59,45 +60,51 @@ export interface DuckDBWasmDrizzleDatabase<
$client: TClient
}
async function getBundles(importUrl = false): Promise<DuckDBBundles> {
const env = await getEnvironment()
switch (env) {
case 'browser':
return importUrl
? (await import('@proj-airi/duckdb-wasm/bundles/import-url-browser')).getImportUrlBundles()
: (await import('@proj-airi/duckdb-wasm/bundles/default-browser')).getBundles()
case 'node':
return importUrl
? await (await import('@proj-airi/duckdb-wasm/bundles/import-url-node')).getImportUrlBundles()
: await (await import('@proj-airi/duckdb-wasm/bundles/default-node')).getBundles()
default:
throw new Error(`Unsupported environment: "${env}"`)
}
}
function constructByDSN<
TSchema extends Record<string, unknown> = Record<string, never>,
>(
dsn: string,
drizzleConfig?: DrizzleConfig<TSchema>,
): DuckDBWasmDrizzleDatabase<TSchema, Promise<DuckDBWasmClient>> {
const structured = parseDSN(dsn)
return construct(connect({
bundles: getBundles(structured.bundles === 'import-url'),
logger: structured.logger ? new ConsoleLogger() : undefined,
storage: structured.storage,
}), drizzleConfig) as any
}
export function drizzle<
TSchema extends Record<string, unknown> = Record<string, never>,
TClient extends Promise<DuckDBWasmClient> = Promise<DuckDBWasmClient>,
>(
...params:
| [{ connection: string | ({ url?: string, bundles?: DuckDBBundles | Promise<DuckDBBundles>, logger?: Logger | false }) }]
| [{ connection: string | ({ url?: string, bundles?: DuckDBBundles | Promise<DuckDBBundles>, logger?: Logger | false }) }, DrizzleConfig<TSchema>]
| [{ connection: string | ConnectOptions }]
| [{ connection: string | ConnectOptions }, DrizzleConfig<TSchema>]
| [{ client: TClient }]
| [{ client: TClient }, DrizzleConfig<TSchema>]
| [ TClient | string ]
| [ TClient | string, DrizzleConfig<TSchema> ]
): DuckDBWasmDrizzleDatabase<TSchema, TClient> {
if (typeof params[0] === 'string') {
const parsedDSN = new URL(params[0] as string)
if (parsedDSN.searchParams.get('bundles') === 'import-url') {
const logger = parsedDSN.searchParams.get('logger') === 'true' ? new ConsoleLogger() : undefined
return construct(new Promise<DuckDBWasmClient>((resolve) => {
getEnvironment().then((env) => {
if (env === 'browser') {
import('@proj-airi/duckdb-wasm/bundles/import-url-browser')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else if (env === 'node') {
import('@proj-airi/duckdb-wasm/bundles/import-url-node')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else {
throw new Error('Unsupported environment')
}
})
}), params[1]) as any
}
const instance = connect({})
return construct(instance, params[1]) as any
return constructByDSN(params[0] as string, params[1]) as any
}
if (isConfig(params[0])) {
@@ -106,48 +113,21 @@ export function drizzle<
client,
...drizzleConfig
} = params[0] as {
connection?: {
url?: string
bundles?: DuckDBBundles
}
connection?: string | ConnectOptions // a DSN or a ConnectOptions object
client?: TClient
} & DrizzleConfig<TSchema>
if (client)
return construct(client, drizzleConfig) as any
if (typeof connection === 'object') {
if (connection.url !== undefined) {
const { url } = connection
const parsedDSN = new URL(url)
const logger = parsedDSN.searchParams.get('logger') === 'true' ? new ConsoleLogger() : undefined
if (parsedDSN.searchParams.get('bundles') === 'import-url') {
return construct(new Promise<DuckDBWasmClient>((resolve) => {
getEnvironment().then((env) => {
if (env === 'browser') {
import('@proj-airi/duckdb-wasm/bundles/import-url-browser')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else if (env === 'node') {
import('@proj-airi/duckdb-wasm/bundles/import-url-node')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else {
throw new Error('Unsupported environment')
}
})
}), drizzleConfig) as any
}
}
if (typeof connection === 'string')
return constructByDSN(connection, drizzleConfig) as any
return construct(connect({ bundles: connection.bundles }), drizzleConfig) as any
}
return construct(connect({}), drizzleConfig) as any
return construct(connect({
bundles: connection.bundles,
logger: connection.logger,
storage: connection.storage,
}), drizzleConfig) as any
}
return construct(params[0] as TClient, params[1] as DrizzleConfig<TSchema> | undefined) as any
@@ -0,0 +1,142 @@
import type { DBOriginPrivateFS } from '@proj-airi/duckdb-wasm'
import type { StructuredDSN } from './dsn'
import { DuckDBAccessMode } from '@duckdb/duckdb-wasm'
import { DBStorageType } from '@proj-airi/duckdb-wasm'
import { describe, expect, it } from 'vitest'
import { buildDSN, parseDSN } from './dsn'
import { spyConsoleWarn } from './test-utils'
describe('parseDSN', { timeout: 10000 }, async () => {
it('should fail with non-duckdb-wasm protocol', async () => {
const dsn = 'random-db:///database.db'
expect(() => parseDSN(dsn)).toThrow('Expected scheme to be "duckdb-wasm:" but got "random-db:"')
})
it('should parse OPFS with path', async () => {
const dsn = 'duckdb-wasm:///path/to/database.db?storage=origin-private-fs'
let structured: StructuredDSN
expect(() => structured = parseDSN(dsn)).not.toThrow()
expect(structured).toEqual({
scheme: 'duckdb-wasm:',
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: 'path/to/database.db',
} as DBOriginPrivateFS,
})
})
it('should parse OPFS with missing leading slash', async () => {
const consoleWarnMock = spyConsoleWarn()
const dsn = 'duckdb-wasm://path/to/database.db?storage=origin-private-fs'
// ^~~~~ missing leading slash: this will be parsed as host/hostname
let structured: StructuredDSN
expect(() => structured = parseDSN(dsn)).not.toThrow()
expect(structured).toEqual({
scheme: 'duckdb-wasm:',
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: 'to/database.db',
} as DBOriginPrivateFS,
})
expect(consoleWarnMock).toHaveBeenCalledTimes(1)
expect(consoleWarnMock).toHaveBeenCalledWith('Host "path" will be ignored while using Origin Private FS')
})
it('should parse OPFS with path and (write = true)', async () => {
const dsn = 'duckdb-wasm:///path/to/database.db?storage=origin-private-fs&write=true'
const structured = parseDSN(dsn)
expect(structured).toEqual({
scheme: 'duckdb-wasm:',
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: 'path/to/database.db',
accessMode: DuckDBAccessMode.READ_WRITE,
} as DBOriginPrivateFS,
})
})
it('should parse OPFS with path and (write = false)', async () => {
const dsn = 'duckdb-wasm:///path/to/database.db?storage=origin-private-fs&write=false'
const structured = parseDSN(dsn)
expect(structured).toEqual({
scheme: 'duckdb-wasm:',
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: 'path/to/database.db',
} as DBOriginPrivateFS,
})
})
})
describe('buildDSN', { timeout: 10000 }, async () => {
it('should build DSN with OPFS (ro)', async () => {
const structured: StructuredDSN = {
scheme: 'duckdb-wasm:',
bundles: 'import-url',
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: 'path/to/database.db',
} as DBOriginPrivateFS,
}
let url: URL
expect(() => url = new URL(buildDSN(structured))).not.toThrow()
expect(url.protocol).toBe('duckdb-wasm:')
expect(url.host).toBe('')
expect(url.pathname).toBe('/path/to/database.db')
expect(url.searchParams.get('storage')).toBe(DBStorageType.ORIGIN_PRIVATE_FS)
expect(url.searchParams.get('write')).toBeNull()
expect(url.searchParams.get('bundles')).toBe('import-url')
})
it('should build DSN with OPFS (rw)', async () => {
const structured: StructuredDSN = {
scheme: 'duckdb-wasm:',
bundles: 'import-url',
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: 'path/to/database.db',
accessMode: DuckDBAccessMode.READ_WRITE,
} as DBOriginPrivateFS,
}
let url: URL
expect(() => url = new URL(buildDSN(structured))).not.toThrow()
expect(url.protocol).toBe('duckdb-wasm:')
expect(url.host).toBe('')
expect(url.pathname).toBe('/path/to/database.db')
expect(url.searchParams.get('storage')).toBe(DBStorageType.ORIGIN_PRIVATE_FS)
expect(url.searchParams.get('write')).toBe('true')
expect(url.searchParams.get('bundles')).toBe('import-url')
})
it('should build the same DSN with OPFS but with a leading slash in the path', async () => {
const structured: StructuredDSN = {
scheme: 'duckdb-wasm:',
bundles: 'import-url',
storage: {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: '/path/to/database.db',
accessMode: DuckDBAccessMode.READ_WRITE,
} as DBOriginPrivateFS,
}
let url: URL
expect(() => url = new URL(buildDSN(structured))).not.toThrow()
expect(url.protocol).toBe('duckdb-wasm:')
expect(url.host).toBe('')
expect(url.pathname).toBe('/path/to/database.db')
expect(url.searchParams.get('storage')).toBe(DBStorageType.ORIGIN_PRIVATE_FS)
expect(url.searchParams.get('write')).toBe('true')
expect(url.searchParams.get('bundles')).toBe('import-url')
})
})
+117
View File
@@ -0,0 +1,117 @@
import type { DBStorage } from '@proj-airi/duckdb-wasm'
import { DuckDBAccessMode } from '@duckdb/duckdb-wasm'
import { DBStorageType } from '@proj-airi/duckdb-wasm'
export interface StructuredDSN {
scheme: 'duckdb-wasm:'
bundles?: 'import-url'
logger?: boolean
storage?: DBStorage
}
export function isLiterallyTrue(value?: string): boolean {
return typeof value === 'string' && /^true$/i.test(value)
}
/**
* Parse a DuckDB WASM DSN string into a structured object
*
* Examples:
* - `duckdb-wasm:///` -> In-memory
* - `duckdb-wasm:///?bundles=import-url` -> In-memory, use import-URL bundles
* - `duckdb-wasm:///?logger=true` -> In-memory, enable logger
* - `duckdb-wasm://database.db?storage=origin-private-fs&write=true` -> Origin Private FS, RW, database.db
* - `duckdb-wasm:///database.db?storage=origin-private-fs&write=true` -> Origin Private FS, RW, database.db (leading slash is optional)
*
* @param dsn The DSN string to parse
*/
export function parseDSN(dsn: string): StructuredDSN {
const structured: StructuredDSN = {
scheme: 'duckdb-wasm:',
}
const parsed = new URL(dsn)
// The protocol in the URL maps to the scheme in the DSN (URI)
// See: https://developer.mozilla.org/en-US/docs/Web/API/URL/protocol
if (!parsed.protocol.startsWith('duckdb-wasm:')) {
throw new Error(`Expected scheme to be "duckdb-wasm:" but got "${parsed.protocol}"`)
}
if (parsed.searchParams.get('bundles') === 'import-url') {
structured.bundles = 'import-url'
}
if (isLiterallyTrue(parsed.searchParams.get('logger'))) {
structured.logger = true
}
const paramStorage = parsed.searchParams.get('storage')
switch (paramStorage) {
case DBStorageType.ORIGIN_PRIVATE_FS: {
if (parsed.host.length > 0) {
console.warn(`Host "${parsed.host}" will be ignored while using Origin Private FS`)
}
const paramWrite = parsed.searchParams.get('write')
structured.storage = {
type: DBStorageType.ORIGIN_PRIVATE_FS,
path: parsed.pathname.startsWith('/') ? parsed.pathname.slice(1) : parsed.pathname,
...isLiterallyTrue(paramWrite) && {
accessMode: DuckDBAccessMode.READ_WRITE,
},
}
break
}
case null:
break
default:
console.warn(`Unknown storage type "${paramStorage}"`)
break
}
return structured
}
/**
* Build a DuckDB WASM DSN string from a structured DSN object
*
* @param structured The structured DSN object
* @returns The DSN string
*/
export function buildDSN(structured: StructuredDSN): string {
const parsed = new URL('duckdb-wasm:///')
if (structured.bundles === 'import-url') {
parsed.searchParams.set('bundles', 'import-url')
}
if (structured.logger) {
parsed.searchParams.set('logger', 'true')
}
if (structured.storage) {
parsed.searchParams.set('storage', structured.storage.type)
switch (structured.storage.type) {
case DBStorageType.ORIGIN_PRIVATE_FS:
parsed.pathname = structured.storage.path
if (!parsed.pathname.startsWith('/')) {
// To make the pathname pathname in the URL
parsed.pathname = `/${parsed.pathname}`
}
if (structured.storage.accessMode === DuckDBAccessMode.READ_WRITE) {
parsed.searchParams.set('write', 'true')
}
break
case DBStorageType.NODE_FS:
parsed.pathname = structured.storage.path
if (structured.storage.accessMode === DuckDBAccessMode.READ_WRITE) {
parsed.searchParams.set('write', 'true')
}
break
}
}
return parsed.toString()
}
@@ -1,13 +0,0 @@
import { getBundles } from '@proj-airi/duckdb-wasm/bundles/default-node'
import { describe, expect, it } from 'vitest'
import { drizzle } from '.'
describe('drizzle', { timeout: 10000 }, async () => {
it('should connect to a DuckDBWasm database', async () => {
const db = drizzle({ connection: { bundles: getBundles() } })
const res = await db.execute('SELECT count(*)::INTEGER as v FROM generate_series(0, 100) t(v)')
expect(res).toBeDefined()
expect(res).toEqual([{ v: 101 }])
})
})
@@ -0,0 +1,12 @@
import { afterAll, vi } from 'vitest'
export function spyConsoleWarn() {
const hookedConsoleWarn = console.warn
const consoleWarnMock = vi.spyOn(console, 'warn').mockImplementation(hookedConsoleWarn)
afterAll(() => {
consoleWarnMock.mockReset()
})
return consoleWarnMock
}
+3 -1
View File
@@ -5,12 +5,14 @@
"ESNext",
"DOM",
"DOM.Iterable",
"DOM.AsyncIterable",
"WebWorker"
],
"module": "ESNext",
"moduleResolution": "bundler",
"types": [
"vite/client"
"vite/client",
"@vitest/browser/providers/playwright"
],
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
@@ -0,0 +1,32 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
test: {
workspace: [
{
extends: true,
test: {
name: 'node',
environment: 'node',
include: ['**/*.{spec,test}.ts'],
exclude: ['**/*.browser.{spec,test}.ts', '**/node_modules/**'],
},
},
{
extends: true,
test: {
name: 'browser',
include: ['**/*.browser.{spec,test}.ts'],
exclude: ['**/node_modules/**'],
browser: {
enabled: true,
provider: 'playwright',
instances: [
{ browser: 'chromium' },
],
},
},
},
],
},
})