nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(memory-driver-duckdb): support Node.js environment

Browse Source
This commit is contained in:
Neko Ayaka
2025-02-17 11:34:17 +08:00
parent 115e91eaf2
commit e989f5e9c9
14 changed files with 764 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cspell.config.yaml
+16
View File
@@ -11,6 +11,8 @@ words:
- audioworklet
- awilix
- Ayaka
- baichuan
- baiducloud
- bigserial
- Bitstream
- bumpp
@@ -44,16 +46,20 @@ words:
- elevenlabs
- formkit
- gcornut
- giteeai
- gltf
- grammyjs
- groq
- guiiai
- hfspaceup
- hfsup
- hfup
- higress
- histoire
- hiyori
- huggingface
- huggingspace
- hunyuan
- hyoban
- iconify
- intlify
@@ -79,14 +85,17 @@ words:
- neuri
- Neuro
- Neuro-sama
- novita
- nuxi
- nuxt
- nuxtjs
- ofetch
- ollama
- Oneliner
- onnx
- onnxruntime
- openai
- openrouter
- opusscript
- pgvector
- picklist
@@ -97,17 +106,22 @@ words:
- pretrained
- prismarine
- pthread
- qwen
- rehype
- rushstack
- Saccade
- saccades
- SAVEPOINT
- sensenova
- Shadcn
- shiki
- shikijs
- silero
- siliconcloud
- sizecheck
- smallserial
- Sniglet
- stepfun
- supergroup
- superjson
- tamagotchi
@@ -120,6 +134,7 @@ words:
- valibot
- vaul
- VITE
- vllm
- vrma
- vueuse
- wavefile
@@ -127,5 +142,6 @@ words:
- worklet
- Xenova
- xsai
- zhipu
ignoreWords: []
import: []
packages/memory-driver-duckdb/package.json
+27 -6
View File
@@ -26,10 +26,23 @@
"import": "./dist/drizzle-orm/index.mjs",
"node": "./dist/drizzle-orm/index.cjs"
},
"./drizzle-orm/vite": {
"types": "./dist/drizzle-orm/vite.d.ts",
"import": "./dist/drizzle-orm/vite.mjs",
"node": "./dist/drizzle-orm/vite.cjs"
"./drizzle-orm/bundles/default-browser": {
"types": "./dist/drizzle-orm/bundles/default-browser.d.ts",
"import": "./dist/drizzle-orm/bundles/default-browser.mjs"
},
"./drizzle-orm/bundles/default-node": {
"types": "./dist/drizzle-orm/bundles/default-node.d.ts",
"import": "./dist/drizzle-orm/bundles/default-node.mjs",
"node": "./dist/drizzle-orm/bundles/default-node.cjs"
},
"./drizzle-orm/bundles/import-url-browser": {
"types": "./dist/drizzle-orm/bundles/import-url-browser.d.ts",
"import": "./dist/drizzle-orm/bundles/import-url-browser.mjs"
},
"./drizzle-orm/bundles/import-url-node": {
"types": "./dist/drizzle-orm/bundles/import-url-node.d.ts",
"import": "./dist/drizzle-orm/bundles/import-url-node.mjs",
"node": "./dist/drizzle-orm/bundles/import-url-node.cjs"
}
},
"main": "./dist/index.cjs",
@@ -50,6 +63,14 @@
"typecheck": "tsc --noEmit",
"db:generate": "drizzle-kit generate"
},
"peerDependencies": {
"web-worker": "^1.5.0"
},
"peerDependenciesMeta": {
"web-worker": {
"optional": true
}
},
"dependencies": {
"@date-fns/tz": "^1.2.0",
"@duckdb/duckdb-wasm": "^1.29.0",
@@ -57,14 +78,14 @@
"date-fns": "^4.1.0",
"defu": "^6.1.4",
"drizzle-orm": "^0.39.2",
"es-toolkit": "^1.32.0",
"superjson": "^2.2.2"
"es-toolkit": "^1.32.0"
},
"devDependencies": {
"@unocss/reset": "^65.4.3",
"@vitejs/plugin-vue": "^5.2.1",
"@vueuse/core": "^12.5.0",
"drizzle-kit": "^0.30.4",
"superjson": "^2.2.2",
"vite": "^6.1.0",
"vue": "^3.5.13",
"vue-tsc": "^2.2.0"
packages/memory-driver-duckdb/src/drizzle-orm/dialect/duckdb-default-bundles.ts → packages/memory-driver-duckdb/src/drizzle-orm/bundles/default-browser.ts
View File
packages/memory-driver-duckdb/src/drizzle-orm/bundles/default-node.ts
+19
View File
@@ -0,0 +1,19 @@
import type { DuckDBBundles } from '@duckdb/duckdb-wasm'
export async function getBundles(): Promise<DuckDBBundles> {
const { createRequire } = await import('node:module')
const { dirname, resolve } = await import('node:path')
const require = createRequire(import.meta.url)
const DUCKDB_DIST = dirname(require.resolve('@duckdb/duckdb-wasm'))
return {
mvp: {
mainModule: resolve(DUCKDB_DIST, './duckdb-mvp.wasm'),
mainWorker: resolve(DUCKDB_DIST, './duckdb-node-mvp.worker.cjs'),
},
eh: {
mainModule: resolve(DUCKDB_DIST, './duckdb-eh.wasm'),
mainWorker: resolve(DUCKDB_DIST, './duckdb-node-eh.worker.cjs'),
},
}
}
packages/memory-driver-duckdb/src/drizzle-orm/dialect/duckdb-vite-bundles.ts → packages/memory-driver-duckdb/src/drizzle-orm/bundles/import-url-browser.ts
+1 -1
View File
@@ -9,7 +9,7 @@ import coiMainModule from '@duckdb/duckdb-wasm/dist/duckdb-coi.wasm?url'
import coiMainWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-coi.worker.js?url'
import coiPthreadWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-coi.pthread.worker.js?url'
export function getViteBundles(): DuckDBBundles {
export function getImportUrlBundles(): DuckDBBundles {
return {
mvp: {
mainModule: mvpMainModule,
packages/memory-driver-duckdb/src/drizzle-orm/bundles/import-url-node.ts
+28
View File
@@ -0,0 +1,28 @@
/* eslint-disable perfectionist/sort-imports */
import type { DuckDBBundles } from '@duckdb/duckdb-wasm'
import ehMainWorker from '@duckdb/duckdb-wasm/dist/duckdb-node-eh.worker.cjs?url'
import ehMainModule from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url'
import mvpMainWorker from '@duckdb/duckdb-wasm/dist/duckdb-node-mvp.worker.cjs?url'
import mvpMainModule from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url'
function transformUrl(url: string) {
if (url.startsWith('/@fs/')) {
return url.replace('/@fs/', 'file://')
}
return url
}
export async function getImportUrlBundles(): Promise<DuckDBBundles> {
return {
mvp: {
mainModule: transformUrl(mvpMainModule),
mainWorker: transformUrl(mvpMainWorker),
},
eh: {
mainModule: transformUrl(ehMainModule),
mainWorker: transformUrl(ehMainWorker),
},
}
}
packages/memory-driver-duckdb/src/drizzle-orm/dialect/duckdb-common.ts
+22
View File
@@ -17,3 +17,25 @@ export function notNullOrUndefined<T>(
): value is T {
return <T>value !== null && <T>value !== undefined
}
export async function getEnvironment() {
if (typeof window !== 'undefined' && window !== null) {
return 'browser'
}
try {
const process = await import('node:process')
if (typeof process !== 'undefined'
&& process.versions != null
&& process.versions.node != null) {
return 'node'
}
return 'unknown'
}
catch {
return 'unknown'
}
return 'unknown'
}
packages/memory-driver-duckdb/src/drizzle-orm/dialect/duckdb.ts
+34 -6
View File
@@ -1,14 +1,14 @@
import type { AsyncDuckDBConnection, DuckDBBundles, Logger } from '@duckdb/duckdb-wasm'
import type { AsyncDuckDBConnection, DuckDBBundle, DuckDBBundles, Logger } from '@duckdb/duckdb-wasm'
import { AsyncDuckDB, ConsoleLogger, selectBundle, VoidLogger } from '@duckdb/duckdb-wasm'
import { defu } from 'defu'
import { getBundles } from './duckdb-default-bundles'
import { getEnvironment } from './duckdb-common'
export type ConnectOptions = ConnectRequiredOptions & ConnectOptionalOptions
export interface ConnectOptionalOptions {
bundles?: DuckDBBundles
bundles?: DuckDBBundles | Promise<DuckDBBundles>
logger?: boolean | Logger
}
@@ -24,10 +24,38 @@ export interface DuckDBWasmClient {
}
export async function connect(options: ConnectOptions): Promise<DuckDBWasmClient> {
const opts = defu(options, { bundles: getBundles(), logger: false })
const opts = defu(options, { logger: false })
const bundle = await selectBundle(opts.bundles)
const worker = new Worker(bundle.mainWorker!)
let worker: Worker
let bundle: DuckDBBundle
const env = await getEnvironment()
if (env === 'browser') {
if (typeof opts.bundles === 'undefined') {
const { getBundles } = await import('../bundles/default-browser')
opts.bundles = await getBundles()
}
bundle = await selectBundle(await opts.bundles)
worker = new Worker(bundle.mainWorker!)
}
else if (env === 'node') {
if (typeof opts.bundles === 'undefined') {
const { getBundles } = await import('../bundles/default-node')
opts.bundles = await getBundles()
}
bundle = await selectBundle(await opts.bundles)
let workerUrl = bundle.mainWorker!
if (workerUrl.startsWith('/@fs/')) {
workerUrl = workerUrl.replace('/@fs/', 'file://')
}
const ww = await import('web-worker')
// eslint-disable-next-line new-cap
worker = new ww.default(workerUrl, { type: 'module' })
}
let logger: Logger
if (opts.logger === true) {
packages/memory-driver-duckdb/src/drizzle-orm/dialect/index.ts
-2
View File
@@ -1,6 +1,4 @@
export * from './duckdb'
export * from './duckdb-common'
export * from './duckdb-default-bundles'
export * from './duckdb-format'
export * from './duckdb-types'
export * from './duckdb-vite-bundles'
packages/memory-driver-duckdb/src/drizzle-orm/driver.ts
+50 -18
View File
@@ -7,7 +7,7 @@ import { ConsoleLogger } from '@duckdb/duckdb-wasm'
import { createTableRelationsHelpers, DefaultLogger, entityKind, extractTablesRelationalConfig, isConfig } from 'drizzle-orm'
import { PgDatabase, PgDialect } from 'drizzle-orm/pg-core'
import { connect } from './dialect'
import { connect, getEnvironment } from './dialect'
import { DuckDBWasmSession } from './session'
export class DuckDBWasmDatabase<
@@ -64,19 +64,35 @@ export function drizzle<
TClient extends Promise<DuckDBWasmClient> = Promise<DuckDBWasmClient>,
>(
...params:
[ TClient | string ] |
[ TClient | string, DrizzleConfig<TSchema> ] |
[(DrizzleConfig<TSchema> & ({ connection: string | ({ url?: string, bundles?: DuckDBBundles, logger?: Logger }) } | { client: TClient })) ]
| [{ connection: string | ({ url?: string, bundles?: DuckDBBundles | Promise<DuckDBBundles>, logger?: Logger | false }) }]
| [{ connection: string | ({ url?: string, bundles?: DuckDBBundles | Promise<DuckDBBundles>, logger?: Logger | false }) }, DrizzleConfig<TSchema>]
| [{ client: TClient }]
| [{ client: TClient }, DrizzleConfig<TSchema>]
| [ TClient | string ]
| [ TClient | string, DrizzleConfig<TSchema> ]
): DuckDBWasmDrizzleDatabase<TSchema, TClient> {
if (typeof params[0] === 'string') {
const parsedDSN = new URL(params[0] as string)
if (parsedDSN.searchParams.get('bundles') === 'import-url') {
const logger = parsedDSN.searchParams.get('logger') === 'true' ? new ConsoleLogger() : undefined
return construct(new Promise<DuckDBWasmClient>((resolve) => {
import('./dialect/duckdb-vite-bundles')
.then(res => res.getViteBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
getEnvironment().then((env) => {
if (env === 'browser') {
import('./bundles/import-url-browser')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else if (env === 'node') {
import('./bundles/import-url-node')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else {
throw new Error('Unsupported environment')
}
})
}), params[1]) as any
}
@@ -100,16 +116,32 @@ export function drizzle<
if (client)
return construct(client, drizzleConfig) as any
if (typeof connection === 'object' && connection.url !== undefined) {
const { url } = connection
const parsedDSN = new URL(url)
if (parsedDSN.searchParams.get('bundles') === 'import-url') {
return construct(new Promise<DuckDBWasmClient>((resolve) => {
import('./dialect/duckdb-vite-bundles')
.then(res => res.getViteBundles())
.then(bundles => connect({ bundles }))
.then(resolve)
}), drizzleConfig) as any
if (typeof connection === 'object') {
if (connection.url !== undefined) {
const { url } = connection
const parsedDSN = new URL(url)
const logger = parsedDSN.searchParams.get('logger') === 'true' ? new ConsoleLogger() : undefined
if (parsedDSN.searchParams.get('bundles') === 'import-url') {
return construct(new Promise<DuckDBWasmClient>((resolve) => {
getEnvironment().then((env) => {
if (env === 'browser') {
import('./bundles/import-url-browser')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else if (env === 'node') {
import('./bundles/import-url-node')
.then(res => res.getImportUrlBundles())
.then(bundles => connect({ bundles, logger }))
.then(resolve)
}
else {
throw new Error('Unsupported environment')
}
})
}), drizzleConfig) as any
}
}
return construct(connect({ bundles: connection.bundles }), drizzleConfig) as any
packages/memory-driver-duckdb/src/drizzle-orm/index.test.ts
+13
View File
@@ -0,0 +1,13 @@
import { describe, expect, it } from 'vitest'
import { drizzle } from '.'
import { getBundles } from './bundles/default-node'
describe('drizzle', { timeout: 10000 }, async () => {
it('should connect to a DuckDBWasm database', async () => {
const db = drizzle({ connection: { bundles: getBundles() } })
const res = await db.execute('SELECT count(*)::INTEGER as v FROM generate_series(0, 100) t(v)')
expect(res).toBeDefined()
expect(res).toEqual([{ v: 101 }])
})
})
packages/memory-driver-duckdb/src/drizzle-orm/index.ts
+1 -1
View File
@@ -1,5 +1,5 @@
export type { ConnectOptionalOptions, ConnectOptions, ConnectRequiredOptions, DuckDBWasmClient } from './dialect'
export { connect, getBundles, mapColumnData } from './dialect'
export { connect, mapColumnData } from './dialect'
export * from './driver'
export * from './migrator'
export * from './session'
packages/memory-driver-duckdb/src/drizzle-orm/vite.ts
-1
View File
@@ -1 +0,0 @@
export { getViteBundles } from './dialect'
pnpm-lock.yaml
Generated
+553 -48
View File
File diff suppressed because it is too large Load Diff