feat: memory driver with @duckdb/duckdb-wasm
This commit is contained in:
Neko Ayaka
@@ -32,6 +32,10 @@ words:
|
||||
- demi
|
||||
- dotenvx
|
||||
- dtype
|
||||
- duckdb
|
||||
- DuckDBWASM
|
||||
- DuckDBWASMQuery
|
||||
- DuckDBWASMQ
|
||||
- elevenlabs
|
||||
- formkit
|
||||
- gcornut
|
||||
@@ -82,6 +86,7 @@ words:
|
||||
- pixi
|
||||
- pixiv
|
||||
- pretrained
|
||||
- pthread
|
||||
- rehype
|
||||
- rushstack
|
||||
- Shadcn
|
||||
|
||||
@@ -9,6 +9,7 @@ export default antfu(
|
||||
'**/assets/js/**',
|
||||
'**/assets/live2d/models/**',
|
||||
'packages/stage-tamagotchi/out/**',
|
||||
'**/drizzle/**',
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
+4
-2
@@ -51,8 +51,10 @@
|
||||
"vitest": "^3.0.4"
|
||||
},
|
||||
"workspaces": [
|
||||
"packages/*",
|
||||
"docs"
|
||||
"packages/**",
|
||||
"services/**",
|
||||
"examples/**",
|
||||
"docs/**"
|
||||
],
|
||||
"simple-git-hooks": {
|
||||
"pre-commit": "pnpm lint-staged && pnpm typecheck"
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { defineConfig } from 'drizzle-kit'
|
||||
|
||||
export default defineConfig({
|
||||
dialect: 'postgresql',
|
||||
schema: './playground/db/schema.ts',
|
||||
out: './playground/drizzle',
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"name": "@proj-airi/memory-driver-duckdb",
|
||||
"type": "module",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "",
|
||||
"author": {
|
||||
"name": "Neko Ayaka",
|
||||
"email": "neko@ayaka.moe",
|
||||
"url": "https://github.com/nekomeowww"
|
||||
},
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/moeru-ai/airi.git",
|
||||
"directory": "packages/memory-driver-duckdb"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.mjs",
|
||||
"require": "./dist/index.cjs"
|
||||
},
|
||||
"./drizzle-orm": {
|
||||
"types": "./dist/drizzle-orm/index.d.ts",
|
||||
"import": "./dist/drizzle-orm/index.mjs",
|
||||
"node": "./dist/drizzle-orm/index.cjs"
|
||||
},
|
||||
"./drizzle-orm/vite": {
|
||||
"types": "./dist/drizzle-orm/vite.d.ts",
|
||||
"import": "./dist/drizzle-orm/vite.mjs",
|
||||
"node": "./dist/drizzle-orm/vite.cjs"
|
||||
}
|
||||
},
|
||||
"main": "./dist/index.cjs",
|
||||
"module": "./dist/index.mjs",
|
||||
"types": "./dist/index.d.ts",
|
||||
"files": [
|
||||
"README.md",
|
||||
"dist",
|
||||
"package.json"
|
||||
],
|
||||
"scripts": {
|
||||
"playground:dev": "vite",
|
||||
"dev": "pnpm run stub && vite",
|
||||
"stub": "unbuild --stub",
|
||||
"build": "unbuild",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"db:generate": "drizzle-kit generate"
|
||||
},
|
||||
"dependencies": {
|
||||
"@date-fns/tz": "^1.2.0",
|
||||
"@duckdb/duckdb-wasm": "^1.29.0",
|
||||
"apache-arrow": "^19.0.0",
|
||||
"date-fns": "^4.1.0",
|
||||
"defu": "^6.1.4",
|
||||
"drizzle-orm": "^0.39.1",
|
||||
"es-toolkit": "^1.32.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@unocss/reset": "^65.4.3",
|
||||
"@vitejs/plugin-vue": "^5.2.1",
|
||||
"@vueuse/core": "^12.5.0",
|
||||
"drizzle-kit": "^0.30.4",
|
||||
"vite": "^6.0.11",
|
||||
"vue": "^3.5.13",
|
||||
"vue-tsc": "^2.2.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { sql } from 'drizzle-orm'
|
||||
import { bigint, pgTable } from 'drizzle-orm/pg-core'
|
||||
|
||||
export const users = pgTable('users', () => {
|
||||
return {
|
||||
id: bigint({ mode: 'number' }).primaryKey().unique().default(sql`0`),
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,4 @@
|
||||
CREATE TABLE "users" (
|
||||
"id" bigint PRIMARY KEY DEFAULT 0 NOT NULL,
|
||||
CONSTRAINT "users_id_unique" UNIQUE("id")
|
||||
);
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"id": "243b9794-66eb-484e-8359-cc3354344cf6",
|
||||
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
"public.users": {
|
||||
"name": "users",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "bigint",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"default": "0"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {
|
||||
"users_id_unique": {
|
||||
"name": "users_id_unique",
|
||||
"nullsNotDistinct": false,
|
||||
"columns": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
}
|
||||
},
|
||||
"enums": {},
|
||||
"schemas": {},
|
||||
"sequences": {},
|
||||
"roles": {},
|
||||
"policies": {},
|
||||
"views": {},
|
||||
"_meta": {
|
||||
"columns": {},
|
||||
"schemas": {},
|
||||
"tables": {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1739028279439,
|
||||
"tag": "0000_blue_selene",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Project AIRI Memory Driver @duckdb/duckdb-wasm Playground</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=0" />
|
||||
<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<script>
|
||||
;(function () {
|
||||
const prefersDark = window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches
|
||||
const setting = localStorage.getItem('vueuse-color-scheme') || 'auto'
|
||||
if (setting === 'dark' || (prefersDark && setting !== 'light'))
|
||||
document.documentElement.classList.toggle('dark', true)
|
||||
})()
|
||||
</script>
|
||||
</head>
|
||||
<body class="font-sans">
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
<noscript> This website requires JavaScript to function properly. Please enable JavaScript to continue. </noscript>
|
||||
</body>
|
||||
</html>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -0,0 +1,8 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.dev/svgjs" width="24" height="24"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24">
|
||||
<g fill="none">
|
||||
<path d="m12.594 23.258l-.012.002l-.071.035l-.02.004l-.014-.004l-.071-.036q-.016-.004-.024.006l-.004.01l-.017.428l.005.02l.01.013l.104.074l.015.004l.012-.004l.104-.074l.012-.016l.004-.017l-.017-.427q-.004-.016-.016-.018m.264-.113l-.014.002l-.184.093l-.01.01l-.003.011l.018.43l.005.012l.008.008l.201.092q.019.005.029-.008l.004-.014l-.034-.614q-.005-.019-.02-.022m-.715.002a.02.02 0 0 0-.027.006l-.006.014l-.034.614q.001.018.017.024l.015-.002l.201-.093l.01-.008l.003-.011l.018-.43l-.003-.012l-.01-.01z"></path>
|
||||
<path fill="#fd7f9c" d="M18.296 3.045a1 1 0 0 1 .657.652l.03.119l1.341 7.154q.154.826.148 1.63l-.012.4H21a1 1 0 0 1 .117 1.993L21 15h-.894q-.164.531-.392 1.033l-.16.33l1.07.856a1 1 0 0 1-1.146 1.634l-.103-.072l-.936-.749A8.43 8.43 0 0 1 12 21a8.42 8.42 0 0 1-6.25-2.755l-.19-.213l-.935.749a1 1 0 0 1-1.343-1.477l.093-.085l1.07-.856a9 9 0 0 1-.435-1.012L3.894 15H3a1 1 0 0 1-.117-1.993L3 13h.54a8.5 8.5 0 0 1 .069-1.619l.067-.411l1.341-7.154a1 1 0 0 1 1.598-.604l.092.08l2.414 2.415a1 1 0 0 0 .576.284L9.828 6h4.344a1 1 0 0 0 .608-.206l.099-.087l2.414-2.414a1 1 0 0 1 1.003-.248m-.93 3.003L16.293 7.12a3 3 0 0 1-2.121.88H9.828a3 3 0 0 1-2.12-.879L6.632 6.048l-.992 5.29A6.5 6.5 0 0 0 5.545 13H7a1 1 0 1 1 0 2h-.492a.998.998 0 0 1 .71 1.696l-.095.086A6.44 6.44 0 0 0 12 19a6.43 6.43 0 0 0 4.696-2.02l.18-.2a1 1 0 0 1 .616-1.78H17a1 1 0 1 1 0-2h1.455a6.5 6.5 0 0 0-.096-1.662zm-3.472 9.005a1 1 0 0 1-.447 1.342l-.553.276a2 2 0 0 1-1.788 0l-.553-.276a1 1 0 0 1 .894-1.79l.553.277l.553-.276a1 1 0 0 1 1.341.447M9.5 10a1.5 1.5 0 1 1 0 3a1.5 1.5 0 0 1 0-3m5 0a1.5 1.5 0 1 1 0 3a1.5 1.5 0 0 1 0-3"></path>
|
||||
</g>
|
||||
</svg><style>@media (prefers-color-scheme: light) { :root { filter: none; } }
|
||||
@media (prefers-color-scheme: dark) { :root { filter: none; } }
|
||||
</style></svg>
|
||||
|
After Width: | Height: | Size: 2.0 KiB |
@@ -0,0 +1,70 @@
|
||||
<script setup lang="ts">
|
||||
import type { DuckDBWasmDrizzleDatabase } from '../../src/drizzle-orm/browser'
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
import { drizzle } from '../../src/drizzle-orm/browser'
|
||||
import * as schema from '../db/schema'
|
||||
import { users } from '../db/schema'
|
||||
import migration from '../migrations/0000_blue_selene.sql?raw'
|
||||
|
||||
const db = ref<DuckDBWasmDrizzleDatabase<typeof schema>>()
|
||||
const results = ref<Record<string, unknown>[]>()
|
||||
const query = ref(`SELECT 1 + 1 AS result`)
|
||||
|
||||
onMounted(async () => {
|
||||
db.value = drizzle('duckdb-wasm://?bundles=worker-url', { schema })
|
||||
await db.value?.execute(migration)
|
||||
|
||||
await db.value.insert(users).values({ id: 1 })
|
||||
const res = await db.value.select().from(users)
|
||||
results.value = res
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
db.value?.$client.then(client => client.close())
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div flex flex-col gap-4 p-4>
|
||||
<h1 text-2xl>
|
||||
<code>@duckdb/duckdb-wasm</code> + <code>drizzle-orm</code> Playground
|
||||
</h1>
|
||||
<div flex flex-col gap-2>
|
||||
<h2 text-xl>
|
||||
Executing
|
||||
</h2>
|
||||
<div>
|
||||
<textarea v-model="query" h-full w-full rounded-lg bg="gray-100 dark:gray-800" p-4 font-mono />
|
||||
</div>
|
||||
</div>
|
||||
<div flex flex-col gap-2>
|
||||
<h2 text-xl>
|
||||
Results
|
||||
</h2>
|
||||
<div whitespace-pre-wrap p-4 font-mono>
|
||||
{{ JSON.stringify(results, null, 2) }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style>
|
||||
html,
|
||||
body,
|
||||
#app {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overscroll-behavior: none;
|
||||
}
|
||||
|
||||
html {
|
||||
background: #fff;
|
||||
transition: all 0.3s ease-in-out;
|
||||
}
|
||||
|
||||
html.dark {
|
||||
background: #121212;
|
||||
color-scheme: dark;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createApp } from 'vue'
|
||||
|
||||
import App from './App.vue'
|
||||
import '@unocss/reset/tailwind.css'
|
||||
import 'uno.css'
|
||||
|
||||
createApp(App)
|
||||
.mount('#app')
|
||||
@@ -0,0 +1,19 @@
|
||||
/**
|
||||
* A type predicate that is true if the given value is either undefined
|
||||
* or null.
|
||||
*/
|
||||
export function isNullOrUndefined<T>(
|
||||
value: T | null | undefined,
|
||||
): value is null | undefined {
|
||||
return <T>value === null || <T>value === undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* A type predicate that is true if the given value is neither undefined
|
||||
* nor null.
|
||||
*/
|
||||
export function notNullOrUndefined<T>(
|
||||
value: T | null | undefined,
|
||||
): value is T {
|
||||
return <T>value !== null && <T>value !== undefined
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import type { DuckDBBundles } from '@duckdb/duckdb-wasm'
|
||||
|
||||
export function getBundles(): DuckDBBundles {
|
||||
return {
|
||||
mvp: {
|
||||
mainModule: './duckdb-mvp.wasm',
|
||||
mainWorker: './duckdb-browser-mvp.worker.js',
|
||||
},
|
||||
eh: {
|
||||
mainModule: './duckdb-eh.wasm',
|
||||
mainWorker: './duckdb-browser-eh.worker.js',
|
||||
},
|
||||
coi: {
|
||||
mainModule: './duckdb-coi.wasm',
|
||||
mainWorker: './duckdb-browser-coi.worker.js',
|
||||
pthreadWorker: './duckdb-browser-coi.pthread.worker.js',
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,552 @@
|
||||
import type { Field, StructRow } from 'apache-arrow'
|
||||
import type { DataType } from './duckdb-types'
|
||||
import { TZDate } from '@date-fns/tz'
|
||||
import { Struct, TimeUnit, util } from 'apache-arrow'
|
||||
import {
|
||||
addDays,
|
||||
addHours,
|
||||
addMilliseconds,
|
||||
addMinutes,
|
||||
addMonths,
|
||||
addQuarters,
|
||||
addSeconds,
|
||||
addWeeks,
|
||||
addYears,
|
||||
format as dateFormat,
|
||||
formatDuration as dateFormatDuration,
|
||||
fromUnixTime,
|
||||
setDay,
|
||||
} from 'date-fns'
|
||||
|
||||
import { trimEnd } from 'es-toolkit'
|
||||
import { isNullOrUndefined, notNullOrUndefined } from './duckdb-common'
|
||||
import {
|
||||
|
||||
isDatetimeType,
|
||||
isDateType,
|
||||
isDecimalType,
|
||||
isDurationType,
|
||||
isFloatType,
|
||||
isIntervalType,
|
||||
isListType,
|
||||
isObjectType,
|
||||
isPeriodType,
|
||||
isTimeType,
|
||||
} from './duckdb-types'
|
||||
|
||||
/**
|
||||
* The frequency strings defined in pandas.
|
||||
* See: https://pandas.pydata.org/docs/user_guide/timeseries.html#period-aliases
|
||||
* Not supported: "N" (nanoseconds), "U" & "us" (microseconds), and "B" (business days).
|
||||
* Reason is that these types are not supported by moment.js, but also they are not
|
||||
* very commonly used in practice.
|
||||
*/
|
||||
type SupportedPandasOffsetType =
|
||||
// yearly frequency:
|
||||
| 'A' // deprecated alias
|
||||
| 'Y'
|
||||
// quarterly frequency:
|
||||
| 'Q'
|
||||
// monthly frequency:
|
||||
| 'M'
|
||||
// weekly frequency:
|
||||
| 'W'
|
||||
// calendar day frequency:
|
||||
| 'D'
|
||||
// hourly frequency:
|
||||
| 'H' // deprecated alias
|
||||
| 'h'
|
||||
// minutely frequency
|
||||
| 'T' // deprecated alias
|
||||
| 'min'
|
||||
// secondly frequency:
|
||||
| 'S' // deprecated alias
|
||||
| 's'
|
||||
// milliseconds frequency:
|
||||
| 'L' // deprecated alias
|
||||
| 'ms'
|
||||
|
||||
type PandasPeriodFrequency =
|
||||
| SupportedPandasOffsetType
|
||||
| `${SupportedPandasOffsetType}-${string}`
|
||||
|
||||
const BASE_DATE = new Date(1970, 0, 1) // 1970-01-01
|
||||
|
||||
function formatMs(duration: number): string {
|
||||
return dateFormat(addMilliseconds(BASE_DATE, duration), 'yyyy-MM-dd HH:mm:ss.SSS')
|
||||
}
|
||||
|
||||
function formatSec(duration: number): string {
|
||||
return dateFormat(addSeconds(BASE_DATE, duration), 'yyyy-MM-dd HH:mm:ss')
|
||||
}
|
||||
|
||||
function formatMin(duration: number): string {
|
||||
return dateFormat(addMinutes(BASE_DATE, duration), 'yyyy-MM-dd HH:mm')
|
||||
}
|
||||
|
||||
function formatHours(duration: number): string {
|
||||
return dateFormat(addHours(BASE_DATE, duration), 'yyyy-MM-dd HH:mm')
|
||||
}
|
||||
|
||||
function formatDay(duration: number): string {
|
||||
return dateFormat(addDays(BASE_DATE, duration), 'yyyy-MM-dd')
|
||||
}
|
||||
|
||||
function formatMonth(duration: number): string {
|
||||
return dateFormat(addMonths(BASE_DATE, duration), 'yyyy-MM')
|
||||
}
|
||||
|
||||
function formatYear(duration: number): string {
|
||||
return dateFormat(addYears(BASE_DATE, duration), 'yyyy')
|
||||
}
|
||||
|
||||
function formatWeeks(duration: number, freqParam?: string): string {
|
||||
if (!freqParam) {
|
||||
throw new Error('Frequency "W" requires parameter')
|
||||
}
|
||||
const WEEKDAY_SHORT = ['SUN', 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT']
|
||||
const dayIndex = WEEKDAY_SHORT.indexOf(freqParam)
|
||||
if (dayIndex < 0) {
|
||||
throw new Error(
|
||||
`Invalid value: ${freqParam}. Supported values: ${JSON.stringify(
|
||||
WEEKDAY_SHORT,
|
||||
)}`,
|
||||
)
|
||||
}
|
||||
|
||||
const weekDate = addWeeks(BASE_DATE, duration)
|
||||
const startDate = dateFormat(setDay(weekDate, dayIndex - 6), 'yyyy-MM-dd')
|
||||
const endDate = dateFormat(setDay(weekDate, dayIndex), 'yyyy-MM-dd')
|
||||
|
||||
return `${startDate}/${endDate}`
|
||||
}
|
||||
|
||||
function formatQuarter(duration: number): string {
|
||||
const date = addQuarters(BASE_DATE, duration)
|
||||
const year = dateFormat(date, 'yyyy')
|
||||
const quarter = Math.floor(date.getMonth() / 3) + 1
|
||||
return `${year}Q${quarter}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Formatters for the different pandas period frequencies.
|
||||
*
|
||||
* This is a mapping from the frequency strings to the function that formats the period.
|
||||
*/
|
||||
const PERIOD_TYPE_FORMATTERS: Record<
|
||||
SupportedPandasOffsetType,
|
||||
(duration: number, freqParam?: string) => string
|
||||
> = {
|
||||
L: formatMs,
|
||||
ms: formatMs,
|
||||
S: formatSec,
|
||||
s: formatSec,
|
||||
T: formatMin,
|
||||
min: formatMin,
|
||||
H: formatHours,
|
||||
h: formatHours,
|
||||
D: formatDay,
|
||||
M: formatMonth,
|
||||
W: formatWeeks,
|
||||
Q: formatQuarter,
|
||||
Y: formatYear,
|
||||
A: formatYear,
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjusts a time value to seconds based on the unit information in the field.
|
||||
*
|
||||
* The unit numbers are specified here:
|
||||
* https://github.com/apache/arrow/blob/3ab246f374c17a216d86edcfff7ff416b3cff803/js/src/enum.ts#L95
|
||||
*
|
||||
* @param timestamp The timestamp to convert.
|
||||
* @param unit The unit of the timestamp. 0 is seconds, 1 is milliseconds, 2 is microseconds, 3 is nanoseconds.
|
||||
* @returns The timestamp in seconds.
|
||||
*/
|
||||
function convertTimestampToSeconds(
|
||||
timestamp: number | bigint,
|
||||
unit: TimeUnit,
|
||||
): number {
|
||||
let unitAdjustment
|
||||
|
||||
if (unit === TimeUnit.MILLISECOND) {
|
||||
// Milliseconds
|
||||
unitAdjustment = 1000
|
||||
}
|
||||
else if (unit === TimeUnit.MICROSECOND) {
|
||||
// Microseconds
|
||||
unitAdjustment = 1000 * 1000
|
||||
}
|
||||
else if (unit === TimeUnit.NANOSECOND) {
|
||||
// Nanoseconds
|
||||
unitAdjustment = 1000 * 1000 * 1000
|
||||
}
|
||||
else {
|
||||
// Interpret it as seconds as a fallback
|
||||
return Number(timestamp)
|
||||
}
|
||||
|
||||
// Do the calculation based on bigints, if the value
|
||||
// is a bigint and not safe for usage as number.
|
||||
// This might lose some precision since it doesn't keep
|
||||
// fractional parts.
|
||||
if (
|
||||
typeof timestamp === 'bigint'
|
||||
&& !Number.isSafeInteger(Number(timestamp))
|
||||
) {
|
||||
return Number(timestamp / BigInt(unitAdjustment))
|
||||
}
|
||||
|
||||
return Number(timestamp) / unitAdjustment
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a UTC time value (timestamp) to a date object.
|
||||
*
|
||||
* @param timestamp The timestamp to convert.
|
||||
* @param field The field containing the unit information.
|
||||
* @returns The date object in UTC timezone.
|
||||
*/
|
||||
export function convertTimeToDate(
|
||||
timestamp: number | bigint,
|
||||
field?: Field,
|
||||
): Date {
|
||||
// Time values from arrow are not converted to a shared unit and
|
||||
// just return the raw arrow value. Therefore, we need to adjust
|
||||
// the value to seconds based on the unit information in the field.
|
||||
// https://github.com/apache/arrow/blob/9e08c57c0986531879aadf7942998d26a94a5d1b/js/src/visitor/get.ts#L193C7-L209
|
||||
const timeInSeconds = convertTimestampToSeconds(
|
||||
timestamp,
|
||||
// The default is SECOND because that is the default unit for time values in pandas.
|
||||
// Though we believe that actually always a unit is populated by arrow.
|
||||
field?.type?.unit ?? TimeUnit.SECOND,
|
||||
)
|
||||
return fromUnixTime(timeInSeconds)
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a duration value based on the unit information in the field.
|
||||
*
|
||||
* @param duration The duration value to format.
|
||||
* @param field The field containing the unit information.
|
||||
* @returns The formatted duration value.
|
||||
*/
|
||||
export function formatDuration(duration: number | bigint, field?: Field): string {
|
||||
// unit: 0 is seconds, 1 is milliseconds, 2 is microseconds, 3 is nanoseconds.
|
||||
return dateFormatDuration({
|
||||
seconds: convertTimestampToSeconds(
|
||||
duration,
|
||||
// The default is NANOSECOND because that is the default unit for duration in pandas.
|
||||
// Though we believe that actually always a unit is populated by arrow.
|
||||
field?.type?.unit ?? TimeUnit.NANOSECOND,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a time value based on the unit information in the field.
|
||||
*
|
||||
* @param timestamp The time value to format.
|
||||
* @param field The field containing the unit information.
|
||||
* @returns The formatted time value.
|
||||
*/
|
||||
export function formatTime(timestamp: number | bigint, field?: Field): string {
|
||||
const date = convertTimeToDate(timestamp, field)
|
||||
return dateFormat(
|
||||
date,
|
||||
date.getMilliseconds() === 0 ? 'HH:mm:ss' : 'HH:mm:ss.SSS',
|
||||
)
|
||||
}
|
||||
|
||||
export function formatDate(date: number | Date): string {
|
||||
// Date values from arrow are already converted to a date object
|
||||
// or a timestamp in milliseconds even if the field unit belonging to the
|
||||
// passed date might have indicated a different unit.
|
||||
// Thats why we don't need the field information here (aka its not passed to the function)
|
||||
// and we don't need to apply any unit conversion.
|
||||
// https://github.com/apache/arrow/blob/9e08c57c0986531879aadf7942998d26a94a5d1b/js/src/visitor/get.ts#L167-L171
|
||||
|
||||
const formatPattern = 'yyyy-MM-dd'
|
||||
|
||||
if (
|
||||
!(
|
||||
date instanceof Date
|
||||
|| (typeof date === 'number' && Number.isFinite(date))
|
||||
)
|
||||
) {
|
||||
console.warn(`Unsupported date value: ${date}`)
|
||||
return String(date)
|
||||
}
|
||||
|
||||
return dateFormat(date, formatPattern)
|
||||
}
|
||||
|
||||
/**
|
||||
* Format datetime value from Arrow to string.
|
||||
*/
|
||||
export function formatDatetime(date: number | Date, field?: Field): string {
|
||||
// Datetime values from arrow are already converted to a date object
|
||||
// or a timestamp in milliseconds even if the field unit might indicate a
|
||||
// different unit.
|
||||
// https://github.com/apache/arrow/blob/9e08c57c0986531879aadf7942998d26a94a5d1b/js/src/visitor/get.ts#L174-L190
|
||||
|
||||
if (
|
||||
!(
|
||||
date instanceof Date
|
||||
|| (typeof date === 'number' && Number.isFinite(date))
|
||||
)
|
||||
) {
|
||||
console.warn(`Unsupported datetime value: ${date}`)
|
||||
return String(date)
|
||||
}
|
||||
|
||||
let datetime: TZDate
|
||||
|
||||
if (typeof date === 'number') {
|
||||
datetime = new TZDate(date, 'UTC')
|
||||
}
|
||||
else {
|
||||
datetime = new TZDate(date, 'UTC')
|
||||
}
|
||||
|
||||
const timezone = field?.type?.timezone
|
||||
if (timezone) {
|
||||
datetime = datetime.withTimeZone(timezone)
|
||||
return dateFormat(datetime, 'yyyy-MM-dd HH:mm:ss OOOO')
|
||||
}
|
||||
// Return the timestamp without timezone information
|
||||
return dateFormat(datetime, 'yyyy-MM-dd HH:mm:ss')
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a decimal value with a given scale to a string.
|
||||
*
|
||||
* This code is partly based on: https://github.com/apache/arrow/issues/35745
|
||||
*
|
||||
* TODO: This is only a temporary workaround until ArrowJS can format decimals correctly.
|
||||
* This is tracked here:
|
||||
* https://github.com/apache/arrow/issues/37920
|
||||
* https://github.com/apache/arrow/issues/28804
|
||||
* https://github.com/apache/arrow/issues/35745
|
||||
*/
|
||||
export function formatDecimal(value: Uint32Array, field?: Field): string {
|
||||
const scale = field?.type?.scale || 0
|
||||
|
||||
// Format Uint32Array to a numerical string and pad it with zeros
|
||||
// So that it is exactly the length of the scale.
|
||||
let numString = util.bigNumToString(new util.BN(value)).padStart(scale, '0')
|
||||
|
||||
// ArrowJS 13 correctly adds a minus sign for negative numbers.
|
||||
// but it doesn't handle th fractional part yet. So we can just return
|
||||
// the value if scale === 0, but we need to do some additional processing
|
||||
// for the fractional part if scale > 0.
|
||||
|
||||
if (scale === 0) {
|
||||
return numString
|
||||
}
|
||||
|
||||
let sign = ''
|
||||
if (numString.startsWith('-')) {
|
||||
// Check if number is negative, and if so remember the sign and remove it.
|
||||
// We will add it back later.
|
||||
sign = '-'
|
||||
numString = numString.slice(1)
|
||||
}
|
||||
// Extract the whole number part. If the number is < 1, it doesn't
|
||||
// have a whole number part, so we'll use "0" instead.
|
||||
// E.g for 123450 with scale 3, we'll get "123" as the whole part.
|
||||
const wholePart = numString.slice(0, -scale) || '0'
|
||||
// Extract the fractional part and remove trailing zeros.
|
||||
// E.g. for 123450 with scale 3, we'll get "45" as the fractional part.
|
||||
const decimalPart = trimEnd(numString.slice(-scale), '0') || ''
|
||||
// Combine the parts and add the sign.
|
||||
return `${sign}${wholePart}${decimalPart ? `.${decimalPart}` : ''}`
|
||||
}
|
||||
|
||||
const formatter = new Intl.NumberFormat('en-US', { style: 'decimal', maximumFractionDigits: 4, minimumFractionDigits: 4, useGrouping: true })
|
||||
|
||||
/**
|
||||
* Formats a float value to a string.
|
||||
*
|
||||
* @param num The float value to format.
|
||||
* @returns The formatted float value.
|
||||
*/
|
||||
export function formatFloat(num: number): string {
|
||||
if (!Number.isFinite(num)) {
|
||||
return String(num)
|
||||
}
|
||||
|
||||
return formatter.format(num)
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats an interval value from arrow to string.
|
||||
*/
|
||||
export function formatInterval(x: StructRow, field?: Field): string {
|
||||
// Serialization for pandas.Interval is provided by Arrow extensions
|
||||
// https://github.com/pandas-dev/pandas/blob/235d9009b571c21b353ab215e1e675b1924ae55c/
|
||||
// pandas/core/arrays/arrow/extension_types.py#L17
|
||||
const extensionName = field && field.metadata.get('ARROW:extension:name')
|
||||
if (extensionName && extensionName === 'pandas.interval') {
|
||||
const extensionMetadata = JSON.parse(
|
||||
field.metadata.get('ARROW:extension:metadata') as string,
|
||||
)
|
||||
const { closed } = extensionMetadata
|
||||
|
||||
const interval = (x as StructRow).toJSON() as {
|
||||
left: number
|
||||
right: number
|
||||
}
|
||||
|
||||
const leftBracket = closed === 'both' || closed === 'left' ? '[' : '('
|
||||
const rightBracket = closed === 'both' || closed === 'right' ? ']' : ')'
|
||||
|
||||
const leftInterval = format(interval.left, (field.type as Struct)?.children?.[0])
|
||||
const rightInterval = format(interval.right, (field.type as Struct)?.children?.[1])
|
||||
|
||||
return `${leftBracket + leftInterval}, ${rightInterval + rightBracket}`
|
||||
}
|
||||
|
||||
return String(x)
|
||||
}
|
||||
|
||||
export function formatPeriodFromFreq(
|
||||
duration: number | bigint,
|
||||
freq: PandasPeriodFrequency,
|
||||
): string {
|
||||
const [freqName, freqParam] = freq.split('-', 2)
|
||||
const momentConverter
|
||||
= PERIOD_TYPE_FORMATTERS[freqName as SupportedPandasOffsetType]
|
||||
if (!momentConverter) {
|
||||
console.warn(`Unsupported period frequency: ${freq}`)
|
||||
return String(duration)
|
||||
}
|
||||
const durationNumber = Number(duration)
|
||||
if (!Number.isSafeInteger(durationNumber)) {
|
||||
console.warn(
|
||||
`Unsupported value: ${duration}. Supported values: [${Number.MIN_SAFE_INTEGER}-${Number.MAX_SAFE_INTEGER}]`,
|
||||
)
|
||||
|
||||
return String(duration)
|
||||
}
|
||||
return momentConverter(durationNumber, freqParam)
|
||||
}
|
||||
|
||||
export function formatPeriod(duration: number | bigint, field?: Field): string {
|
||||
// Serialization for pandas.Period is provided by Arrow extensions
|
||||
// https://github.com/pandas-dev/pandas/blob/70bb855cbbc75b52adcb127c84e0a35d2cd796a9/pandas/core/arrays/arrow/extension_types.py#L26
|
||||
if (isNullOrUndefined(field)) {
|
||||
console.warn('Field information is missing')
|
||||
return String(duration)
|
||||
}
|
||||
|
||||
const extensionName = field.metadata.get('ARROW:extension:name')
|
||||
const extensionMetadata = field.metadata.get('ARROW:extension:metadata')
|
||||
|
||||
if (
|
||||
isNullOrUndefined(extensionName)
|
||||
|| isNullOrUndefined(extensionMetadata)
|
||||
) {
|
||||
console.warn('Arrow extension metadata is missing')
|
||||
return String(duration)
|
||||
}
|
||||
|
||||
if (extensionName !== 'pandas.period') {
|
||||
console.warn(`Unsupported extension name for period type: ${extensionName}`)
|
||||
return String(duration)
|
||||
}
|
||||
|
||||
const parsedExtensionMetadata = JSON.parse(extensionMetadata as string)
|
||||
const { freq } = parsedExtensionMetadata
|
||||
return formatPeriodFromFreq(duration, freq)
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats nested arrays and other objects to a JSON string.
|
||||
*
|
||||
* @param object The value to format.
|
||||
* @param field The field metadata from arrow containing metadata about the column.
|
||||
* @returns The formatted JSON string.
|
||||
*/
|
||||
export function formatObject(object: any, field?: Field): string {
|
||||
if (field?.type instanceof Struct) {
|
||||
// This type is used by python dictionary values
|
||||
|
||||
return JSON.stringify(object, (_key, value) => {
|
||||
if (!notNullOrUndefined(value)) {
|
||||
// Workaround: Arrow JS adds all properties from all cells
|
||||
// as fields. When you convert to string, it will contain lots of fields with
|
||||
// null values. To mitigate this, we filter out null values.
|
||||
return undefined
|
||||
}
|
||||
if (typeof value === 'bigint') {
|
||||
// JSON.stringify fails to serialize bigint values, therefore we have to
|
||||
// handle them manually.
|
||||
// TODO(lukasmasuch): Would it be better to serialize it to a string to
|
||||
// not lose precision?
|
||||
return Number(value)
|
||||
}
|
||||
return value
|
||||
})
|
||||
}
|
||||
|
||||
// TODO(lukasmasuch): Investigate if we can unify this with the logic above.
|
||||
return JSON.stringify(object, (_key, value) =>
|
||||
typeof value === 'bigint' ? Number(value) : value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the cell data and type metadata from arrow and nicely formats it into a human-readable string.
|
||||
*
|
||||
* This is mostly a best-effort logic and should not throw exceptions in case of unknown values
|
||||
* or other issues. This makes it easier to use this method by consumers (table, dataframe) since
|
||||
* they would have to somehow deal with the exception on a cell level to not crash the full table or app.
|
||||
*
|
||||
* @param x The cell value.
|
||||
* @param field The field metadata from arrow containing metadata about the column.
|
||||
* @returns The formatted cell value.
|
||||
*/
|
||||
export function format(x: DataType, field?: Field): string {
|
||||
if (isNullOrUndefined(x)) {
|
||||
return ''
|
||||
}
|
||||
|
||||
const isDate = x instanceof Date || Number.isFinite(x)
|
||||
if (isDate && isDateType(field)) {
|
||||
return formatDate(x as Date | number)
|
||||
}
|
||||
|
||||
if (typeof x === 'bigint' && isTimeType(field)) {
|
||||
return formatTime(Number(x), field)
|
||||
}
|
||||
|
||||
if (isDate && isDatetimeType(field)) {
|
||||
return formatDatetime(x as Date | number, field)
|
||||
}
|
||||
|
||||
if (isPeriodType(field)) {
|
||||
return formatPeriod(x as bigint, field)
|
||||
}
|
||||
|
||||
if (isIntervalType(field)) {
|
||||
return formatInterval(x as StructRow, field)
|
||||
}
|
||||
|
||||
if (isDurationType(field)) {
|
||||
return formatDuration(x as number | bigint, field)
|
||||
}
|
||||
|
||||
if (isDecimalType(field)) {
|
||||
return formatDecimal(x as Uint32Array, field)
|
||||
}
|
||||
|
||||
if (isFloatType(field) && Number.isFinite(x)) {
|
||||
return formatFloat(x as number)
|
||||
}
|
||||
|
||||
if (isObjectType(field) || isListType(field)) {
|
||||
return formatObject(x, field)
|
||||
}
|
||||
|
||||
return String(x)
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
import type { Dictionary, Field, Struct, StructRow, Vector } from 'apache-arrow'
|
||||
import { DataType as ArrowDataType } from 'apache-arrow'
|
||||
import { isNullOrUndefined } from './duckdb-common'
|
||||
|
||||
/** Data types used by ArrowJS. */
|
||||
export type DataType =
|
||||
| null
|
||||
| boolean
|
||||
| number
|
||||
| string
|
||||
| Date // datetime
|
||||
| Int32Array // int
|
||||
| Uint8Array // bytes
|
||||
| Uint32Array // Decimal
|
||||
| Vector // arrays
|
||||
| StructRow // interval
|
||||
| Dictionary // categorical
|
||||
| Struct // dict
|
||||
| bigint // period
|
||||
|
||||
/** The type of the cell. */
|
||||
export enum DataFrameCellType {
|
||||
// Index cells
|
||||
INDEX = 'index',
|
||||
// Data cells
|
||||
DATA = 'data',
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an Arrow vector to a list of strings.
|
||||
*
|
||||
* @param vector The Arrow vector to convert.
|
||||
* @returns The list of strings.
|
||||
*/
|
||||
export function convertVectorToList(vector: Vector<any>): string[] {
|
||||
const values = []
|
||||
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
values.push(vector.get(i))
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
/** Returns the timezone of the arrow type metadata. */
|
||||
export function getTimezone(field: Field): string | undefined {
|
||||
return field.type?.timezone ?? field.metadata?.get('timezone')
|
||||
}
|
||||
|
||||
/**
|
||||
* True if the arrow type is an integer type.
|
||||
* For example: int8, int16, int32, int64, uint8, uint16, uint32, uint64, range
|
||||
*/
|
||||
export function isIntegerType(field?: Field): boolean {
|
||||
if (isNullOrUndefined(field)) {
|
||||
return false
|
||||
}
|
||||
|
||||
return (
|
||||
// Period types are integers with an extra extension name
|
||||
(ArrowDataType.isInt(field.type) && !isPeriodType(field))
|
||||
|| isUnsignedIntegerType(field)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is an unsigned integer type. */
|
||||
export function isUnsignedIntegerType(field?: Field): boolean {
|
||||
if (isNullOrUndefined(field)) {
|
||||
return false
|
||||
}
|
||||
|
||||
return (
|
||||
(ArrowDataType.isInt(field.type)
|
||||
&& field.type.isSigned === false)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* True if the arrow type is a float type.
|
||||
* For example: float16, float32, float64, float96, float128
|
||||
*/
|
||||
export function isFloatType(field?: Field): boolean {
|
||||
if (isNullOrUndefined(field)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
(ArrowDataType.isFloat(field.type))
|
||||
?? false
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a decimal type. */
|
||||
export function isDecimalType(field?: Field): boolean {
|
||||
if (isNullOrUndefined(field)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isDecimal(field.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a numeric type. */
|
||||
export function isNumericType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return isIntegerType(type) || isFloatType(type) || isDecimalType(type)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a boolean type. */
|
||||
export function isBooleanType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isBool(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a duration type. */
|
||||
export function isDurationType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isDuration(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a period type. */
|
||||
export function isPeriodType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
(ArrowDataType.isInt(type.type)
|
||||
&& type.metadata.get('ARROW:extension:name') === 'period')
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a datetime type. */
|
||||
export function isDatetimeType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isTimestamp(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a date type. */
|
||||
export function isDateType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isDate(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a time type. */
|
||||
export function isTimeType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isTime(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a categorical type. */
|
||||
export function isCategoricalType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isDictionary(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a list type. */
|
||||
export function isListType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isList(type.type)
|
||||
|| ArrowDataType.isFixedSizeList(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is an object type. */
|
||||
export function isObjectType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isStruct(type.type)
|
||||
|| ArrowDataType.isMap(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a bytes type. */
|
||||
export function isBytesType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isBinary(type.type)
|
||||
|| ArrowDataType.isLargeBinary(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a string type. */
|
||||
export function isStringType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isUtf8(type.type)
|
||||
|| ArrowDataType.isLargeUtf8(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is an empty type. */
|
||||
export function isEmptyType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
ArrowDataType.isNull(type.type)
|
||||
)
|
||||
}
|
||||
|
||||
/** True if the arrow type is a interval type. */
|
||||
export function isIntervalType(type?: Field): boolean {
|
||||
if (isNullOrUndefined(type)) {
|
||||
return false
|
||||
}
|
||||
// ArrowDataType.isInterval checks for a different (unsupported) type and not related
|
||||
// to the pandas interval extension type.
|
||||
return (
|
||||
(ArrowDataType.isStruct(type.type)
|
||||
&& type.metadata.get('ARROW:extension:name') === 'interval')
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/* eslint-disable perfectionist/sort-imports */
|
||||
import type { DuckDBBundles } from '@duckdb/duckdb-wasm'
|
||||
|
||||
import ehMainWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url'
|
||||
import ehMainModule from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url'
|
||||
|
||||
import mvpMainWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url'
|
||||
import mvpMainModule from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url'
|
||||
|
||||
import coiMainModule from '@duckdb/duckdb-wasm/dist/duckdb-coi.wasm?url'
|
||||
import coiMainWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-coi.worker.js?url'
|
||||
import coiPthreadWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-coi.pthread.worker.js?url'
|
||||
|
||||
export function getViteBundles(): DuckDBBundles {
|
||||
return {
|
||||
mvp: {
|
||||
mainModule: mvpMainModule,
|
||||
mainWorker: mvpMainWorker,
|
||||
},
|
||||
eh: {
|
||||
mainModule: ehMainModule,
|
||||
mainWorker: ehMainWorker,
|
||||
},
|
||||
coi: {
|
||||
mainModule: coiMainModule,
|
||||
mainWorker: coiMainWorker,
|
||||
pthreadWorker: coiPthreadWorker,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import type { AsyncDuckDBConnection, DuckDBBundles, Logger } from '@duckdb/duckdb-wasm'
|
||||
import { AsyncDuckDB, ConsoleLogger, selectBundle, VoidLogger } from '@duckdb/duckdb-wasm'
|
||||
import { defu } from 'defu'
|
||||
import { getBundles } from './duckdb-default-bundles'
|
||||
|
||||
export type ConnectOptions = ConnectRequiredOptions & ConnectOptionalOptions
|
||||
|
||||
export interface ConnectOptionalOptions {
|
||||
bundles?: DuckDBBundles
|
||||
logger?: boolean | Logger
|
||||
}
|
||||
|
||||
export interface ConnectRequiredOptions {
|
||||
|
||||
}
|
||||
|
||||
export interface DuckDBWasmClient {
|
||||
worker: Worker
|
||||
db: AsyncDuckDB
|
||||
conn: AsyncDuckDBConnection
|
||||
close: () => Promise<void>
|
||||
}
|
||||
|
||||
export async function connect(options: ConnectOptions): Promise<DuckDBWasmClient> {
|
||||
const opts = defu(options, { bundles: getBundles(), logger: false })
|
||||
|
||||
const bundle = await selectBundle(opts.bundles)
|
||||
const worker = new Worker(bundle.mainWorker!)
|
||||
|
||||
let logger: Logger
|
||||
if (opts.logger === true) {
|
||||
logger = new ConsoleLogger()
|
||||
}
|
||||
else if (opts.logger === false) {
|
||||
logger = new VoidLogger()
|
||||
}
|
||||
else {
|
||||
logger = opts.logger
|
||||
}
|
||||
|
||||
const db = new AsyncDuckDB(logger, worker)
|
||||
await db.instantiate(bundle.mainModule, bundle.pthreadWorker)
|
||||
const conn = await db.connect()
|
||||
|
||||
return {
|
||||
worker,
|
||||
db,
|
||||
conn,
|
||||
close: async () => {
|
||||
await conn.close()
|
||||
await db.terminate()
|
||||
await worker.terminate()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export async function beginTransaction(client: Promise<DuckDBWasmClient>, txFn: (client: Promise<DuckDBWasmClient>) => Promise<any>): Promise<any> {
|
||||
await (await client).conn.send('BEGIN TRANSACTION')
|
||||
try {
|
||||
const result = await txFn(client)
|
||||
await (await client).conn.send('COMMIT')
|
||||
return result
|
||||
}
|
||||
catch (err) {
|
||||
await (await client).conn.send('ROLLBACK')
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export async function withSavepoint(client: Promise<DuckDBWasmClient>, spName: string, txFn: (client: Promise<DuckDBWasmClient>) => Promise<any>): Promise<any> {
|
||||
await (await client).conn.send(`SAVEPOINT ${spName}`)
|
||||
try {
|
||||
const result = await txFn(client)
|
||||
await (await client).conn.send(`RELEASE SAVEPOINT ${spName}`)
|
||||
return result
|
||||
}
|
||||
catch (err) {
|
||||
await (await client).conn.send(`ROLLBACK TO SAVEPOINT ${spName}`)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export * from './duckdb'
|
||||
export * from './duckdb-common'
|
||||
export * from './duckdb-default-bundles'
|
||||
export * from './duckdb-format'
|
||||
export * from './duckdb-types'
|
||||
export * from './duckdb-vite-bundles'
|
||||
@@ -0,0 +1,140 @@
|
||||
import type { DuckDBBundles, Logger } from '@duckdb/duckdb-wasm'
|
||||
import type { DrizzleConfig, RelationalSchemaConfig, TablesRelationalConfig } from 'drizzle-orm'
|
||||
import type { DuckDBWasmClient } from './dialect'
|
||||
import type { DuckDBWasmQueryResultHKT } from './session'
|
||||
|
||||
import { createTableRelationsHelpers, DefaultLogger, entityKind, extractTablesRelationalConfig, isConfig } from 'drizzle-orm'
|
||||
import { PgDatabase, PgDialect } from 'drizzle-orm/pg-core'
|
||||
import { connect } from './dialect'
|
||||
import { DuckDBWasmSession } from './session'
|
||||
|
||||
export class DuckDBWasmDatabase<
|
||||
TSchema extends Record<string, unknown> = Record<string, never>,
|
||||
> extends PgDatabase<DuckDBWasmQueryResultHKT, TSchema> {
|
||||
static override readonly [entityKind]: string = 'DuckDBWasmDatabase'
|
||||
|
||||
// ** @internal */
|
||||
dialect: PgDialect
|
||||
// ** @internal */
|
||||
session: DuckDBWasmSession<Promise<DuckDBWasmClient>, TSchema, TablesRelationalConfig>
|
||||
}
|
||||
|
||||
function construct<
|
||||
TSchema extends Record<string, unknown> = Record<string, never>,
|
||||
TClient extends Promise<DuckDBWasmClient> = Promise<DuckDBWasmClient>,
|
||||
>(
|
||||
client: Promise<DuckDBWasmClient>,
|
||||
config: DrizzleConfig<TSchema> = {},
|
||||
): DuckDBWasmDrizzleDatabase<TSchema, TClient> {
|
||||
const dialect = new PgDialect({ casing: config.casing })
|
||||
let logger
|
||||
if (config.logger === true) {
|
||||
logger = new DefaultLogger()
|
||||
}
|
||||
else if (config.logger !== false) {
|
||||
logger = config.logger
|
||||
}
|
||||
|
||||
let schema: RelationalSchemaConfig<TablesRelationalConfig> | undefined
|
||||
if (config.schema) {
|
||||
const tablesConfig = extractTablesRelationalConfig(
|
||||
config.schema,
|
||||
createTableRelationsHelpers,
|
||||
)
|
||||
schema = {
|
||||
fullSchema: config.schema,
|
||||
schema: tablesConfig.tables,
|
||||
tableNamesMap: tablesConfig.tableNamesMap,
|
||||
}
|
||||
}
|
||||
|
||||
const session = new DuckDBWasmSession(client, dialect, schema, { logger })
|
||||
const db = new DuckDBWasmDatabase(dialect, session, schema as any) as DuckDBWasmDatabase<TSchema>;
|
||||
(<any>db).$client = client
|
||||
|
||||
return db as any
|
||||
}
|
||||
|
||||
export interface DuckDBWasmDrizzleDatabase<
|
||||
TSchema extends Record<string, unknown> = Record<string, never>,
|
||||
TClient extends Promise<DuckDBWasmClient> = Promise<DuckDBWasmClient>,
|
||||
> extends DuckDBWasmDatabase<TSchema> {
|
||||
$client: TClient
|
||||
}
|
||||
|
||||
export function drizzle<
|
||||
TSchema extends Record<string, unknown> = Record<string, never>,
|
||||
TClient extends Promise<DuckDBWasmClient> = Promise<DuckDBWasmClient>,
|
||||
>(
|
||||
...params:
|
||||
[ TClient | string ] |
|
||||
[ TClient | string, DrizzleConfig<TSchema> ] |
|
||||
[(DrizzleConfig<TSchema> & ({ connection: string | ({ url?: string, bundles?: DuckDBBundles, logger?: Logger }) } | { client: TClient })) ]
|
||||
): DuckDBWasmDrizzleDatabase<TSchema, TClient> {
|
||||
if (typeof params[0] === 'string') {
|
||||
const parsedDSN = new URL(params[0] as string)
|
||||
if (parsedDSN.searchParams.get('bundles') === 'worker-url') {
|
||||
return construct(new Promise<DuckDBWasmClient>((resolve) => {
|
||||
import('./dialect/duckdb-vite-bundles')
|
||||
.then(res => res.getViteBundles())
|
||||
.then(bundles => connect({ bundles }))
|
||||
.then(resolve)
|
||||
}), params[1]) as any
|
||||
}
|
||||
|
||||
const instance = connect({})
|
||||
return construct(instance, params[1]) as any
|
||||
}
|
||||
|
||||
if (isConfig(params[0])) {
|
||||
const {
|
||||
connection,
|
||||
client,
|
||||
...drizzleConfig
|
||||
} = params[0] as {
|
||||
connection?: {
|
||||
url?: string
|
||||
bundles?: DuckDBBundles
|
||||
}
|
||||
client?: TClient
|
||||
} & DrizzleConfig<TSchema>
|
||||
|
||||
if (client)
|
||||
return construct(client, drizzleConfig) as any
|
||||
|
||||
if (typeof connection === 'object' && connection.url !== undefined) {
|
||||
const { url } = connection
|
||||
const parsedDSN = new URL(url)
|
||||
if (parsedDSN.searchParams.get('bundles') === 'worker-url') {
|
||||
return construct(new Promise<DuckDBWasmClient>((resolve) => {
|
||||
import('./dialect/duckdb-vite-bundles')
|
||||
.then(res => res.getViteBundles())
|
||||
.then(bundles => connect({ bundles }))
|
||||
.then(resolve)
|
||||
}), drizzleConfig) as any
|
||||
}
|
||||
|
||||
return construct(connect({ bundles: connection.bundles }), drizzleConfig) as any
|
||||
}
|
||||
|
||||
return construct(connect({}), drizzleConfig) as any
|
||||
}
|
||||
|
||||
return construct(params[0] as TClient, params[1] as DrizzleConfig<TSchema> | undefined) as any
|
||||
}
|
||||
|
||||
// eslint-disable-next-line ts/no-namespace
|
||||
export namespace drizzle {
|
||||
export function mock<TSchema extends Record<string, unknown> = Record<string, never>>(
|
||||
config?: DrizzleConfig<TSchema>,
|
||||
): DuckDBWasmDatabase<TSchema> & {
|
||||
$client: '$client is not available on drizzle.mock()'
|
||||
} {
|
||||
return construct({
|
||||
options: {
|
||||
parsers: {},
|
||||
serializers: {},
|
||||
},
|
||||
} as any, config) as any
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export type { ConnectOptionalOptions, ConnectOptions, ConnectRequiredOptions, DuckDBWasmClient } from './dialect'
|
||||
export { connect, format, getBundles } from './dialect'
|
||||
export * from './driver'
|
||||
export * from './migrator'
|
||||
export * from './session'
|
||||
export type { AsyncDuckDBConnection, DuckDBBundles, Logger } from '@duckdb/duckdb-wasm'
|
||||
export { AsyncDuckDB, ConsoleLogger, selectBundle, VoidLogger } from '@duckdb/duckdb-wasm'
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { MigrationConfig } from 'drizzle-orm/migrator'
|
||||
import type { PgSession } from 'drizzle-orm/pg-core'
|
||||
import type { DuckDBWasmDatabase } from './driver'
|
||||
|
||||
import { readMigrationFiles } from 'drizzle-orm/migrator'
|
||||
|
||||
export async function migrate<TSchema extends Record<string, unknown>>(
|
||||
db: DuckDBWasmDatabase<TSchema>,
|
||||
config: MigrationConfig,
|
||||
) {
|
||||
const migrations = readMigrationFiles(config)
|
||||
await db.dialect.migrate(migrations, db.session as unknown as PgSession, config)
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
import type { Schema, StructRow } from 'apache-arrow'
|
||||
import type { Assume, Logger, Query, RelationalSchemaConfig, TablesRelationalConfig } from 'drizzle-orm'
|
||||
import type { PgDialect, PgQueryResultHKT, PgTransactionConfig, PreparedQueryConfig, SelectedFieldsOrdered } from 'drizzle-orm/pg-core'
|
||||
import type { DuckDBWasmClient } from './dialect'
|
||||
|
||||
import { entityKind, fillPlaceholders, NoopLogger } from 'drizzle-orm'
|
||||
import { PgPreparedQuery, PgSession, PgTransaction } from 'drizzle-orm/pg-core'
|
||||
import { beginTransaction, format, withSavepoint } from './dialect'
|
||||
|
||||
export type Row = Record<string, any>
|
||||
|
||||
export type RowList<T extends Row[]> = T
|
||||
|
||||
function toJSRepresentedRows<T extends { toArray: () => StructRow[], schema: Schema }>(results: T) {
|
||||
const rows = (results.toArray() as StructRow[] || []).map(item => item.toJSON()) || []
|
||||
|
||||
const jsRepresentedRows = rows.map((row) => {
|
||||
results.schema.fields.forEach((field) => {
|
||||
return row[field.name] = format(row[field.name], field)
|
||||
})
|
||||
|
||||
return row
|
||||
})
|
||||
|
||||
return jsRepresentedRows
|
||||
}
|
||||
|
||||
async function callQuery(client: Promise<DuckDBWasmClient>, query: string, params: unknown[]) {
|
||||
const c = await client
|
||||
|
||||
if (!params || params.length === 0) {
|
||||
const results = await c.conn.query(query)
|
||||
return toJSRepresentedRows(results)
|
||||
}
|
||||
|
||||
const stmt = await c.conn.prepare(query)
|
||||
const results = await stmt.query(...params)
|
||||
const rows = toJSRepresentedRows(results)
|
||||
|
||||
stmt.close()
|
||||
return rows
|
||||
}
|
||||
|
||||
export class DuckDBWASMPreparedQuery<T extends PreparedQueryConfig> extends PgPreparedQuery<T> {
|
||||
static override readonly [entityKind]: string = 'DuckDBWasmPreparedQuery'
|
||||
|
||||
constructor(
|
||||
private client: Promise<DuckDBWasmClient>,
|
||||
private queryString: string,
|
||||
private params: unknown[],
|
||||
private logger: Logger,
|
||||
private fields: SelectedFieldsOrdered | undefined,
|
||||
private customResultMapper?: (rows: unknown[][]) => T['execute'],
|
||||
) {
|
||||
super({ sql: queryString, params })
|
||||
}
|
||||
|
||||
async execute(placeholderValues: Record<string, unknown> | undefined = {}): Promise<T['execute']> {
|
||||
const params = fillPlaceholders(this.params, placeholderValues)
|
||||
this.logger.logQuery(this.queryString, params)
|
||||
|
||||
const { fields, queryString: query, client, customResultMapper } = this
|
||||
if (!fields && !customResultMapper) {
|
||||
return callQuery(client, query, params)
|
||||
}
|
||||
|
||||
return callQuery(client, query, params)
|
||||
}
|
||||
|
||||
async all(placeholderValues: Record<string, unknown> | undefined = {}): Promise<T['all']> {
|
||||
const params = fillPlaceholders(this.params, placeholderValues)
|
||||
this.logger.logQuery(this.queryString, params)
|
||||
return callQuery(this.client, this.queryString, params)
|
||||
}
|
||||
}
|
||||
|
||||
export interface DuckDBWASMSessionOptions {
|
||||
logger?: Logger
|
||||
}
|
||||
|
||||
export class DuckDBWasmSession<
|
||||
TSQL extends Promise<DuckDBWasmClient>,
|
||||
TFullSchema extends Record<string, unknown>,
|
||||
TSchema extends TablesRelationalConfig,
|
||||
> extends PgSession<DuckDBWasmQueryResultHKT, TFullSchema, TSchema> {
|
||||
static override readonly [entityKind]: string = 'DuckDBWasmSession'
|
||||
|
||||
logger: Logger
|
||||
|
||||
constructor(
|
||||
public client: TSQL,
|
||||
dialect: PgDialect,
|
||||
private schema: RelationalSchemaConfig<TSchema> | undefined,
|
||||
readonly options: DuckDBWASMSessionOptions = {},
|
||||
) {
|
||||
super(dialect)
|
||||
this.logger = options.logger ?? new NoopLogger()
|
||||
}
|
||||
|
||||
prepareQuery<T extends PreparedQueryConfig = PreparedQueryConfig>(
|
||||
query: Query,
|
||||
fields: SelectedFieldsOrdered | undefined,
|
||||
name: string | undefined,
|
||||
isResponseInArrayMode: boolean,
|
||||
customResultMapper?: (rows: unknown[][]) => T['execute'],
|
||||
): PgPreparedQuery<T> {
|
||||
return new DuckDBWASMPreparedQuery(
|
||||
this.client,
|
||||
query.sql,
|
||||
query.params,
|
||||
this.logger,
|
||||
fields,
|
||||
customResultMapper,
|
||||
)
|
||||
}
|
||||
|
||||
async query(query: string, params: unknown[]): Promise<RowList<Row[]>> {
|
||||
this.logger.logQuery(query, params)
|
||||
return callQuery(this.client, query, params)
|
||||
}
|
||||
|
||||
async queryObjects<T extends Row>(
|
||||
query: string,
|
||||
params: unknown[],
|
||||
): Promise<RowList<T[]>> {
|
||||
this.logger.logQuery(query, params)
|
||||
return callQuery(this.client, query, params) as Promise<RowList<T[]>>
|
||||
}
|
||||
|
||||
override transaction<T>(
|
||||
transaction: (tx: DuckDBWasmTransaction<TFullSchema, TSchema>) => Promise<T>,
|
||||
config?: PgTransactionConfig,
|
||||
): Promise<T> {
|
||||
return beginTransaction(this.client, async (client) => {
|
||||
const session = new DuckDBWasmSession<Promise<DuckDBWasmClient>, TFullSchema, TSchema>(
|
||||
client,
|
||||
this.dialect,
|
||||
this.schema,
|
||||
this.options,
|
||||
)
|
||||
const tx = new DuckDBWasmTransaction(this.dialect, session, this.schema)
|
||||
if (config) {
|
||||
await tx.setTransaction(config)
|
||||
}
|
||||
|
||||
return transaction(tx)
|
||||
}) as Promise<T>
|
||||
}
|
||||
}
|
||||
|
||||
export class DuckDBWasmTransaction<
|
||||
TFullSchema extends Record<string, unknown>,
|
||||
TSchema extends TablesRelationalConfig,
|
||||
> extends PgTransaction<DuckDBWasmQueryResultHKT, TFullSchema, TSchema> {
|
||||
static override readonly [entityKind]: string = 'DuckDBWasmTransaction'
|
||||
dialect: PgDialect
|
||||
session: DuckDBWasmSession<Promise<DuckDBWasmClient>, TFullSchema, TSchema>
|
||||
|
||||
constructor(
|
||||
dialect: PgDialect,
|
||||
session: DuckDBWasmSession<Promise<DuckDBWasmClient>, TFullSchema, TSchema>,
|
||||
schema: RelationalSchemaConfig<TSchema> | undefined,
|
||||
nestedIndex = 0,
|
||||
) {
|
||||
super(dialect, session, schema, nestedIndex)
|
||||
this.dialect = dialect
|
||||
this.session = session
|
||||
}
|
||||
|
||||
override async transaction<T>(
|
||||
transaction: (tx: DuckDBWasmTransaction<TFullSchema, TSchema>) => Promise<T>,
|
||||
): Promise<T> {
|
||||
return withSavepoint(this.session.client, '', async (client) => {
|
||||
const session = new DuckDBWasmSession<Promise<DuckDBWasmClient>, TFullSchema, TSchema>(
|
||||
client,
|
||||
this.dialect,
|
||||
this.schema,
|
||||
this.session.options,
|
||||
)
|
||||
|
||||
const tx = new DuckDBWasmTransaction<TFullSchema, TSchema>(this.dialect, session, this.schema)
|
||||
return transaction(tx)
|
||||
}) as Promise<T>
|
||||
}
|
||||
}
|
||||
|
||||
export interface DuckDBWasmQueryResultHKT extends PgQueryResultHKT {
|
||||
type: RowList<Assume<this['row'], Row>[]>
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { getViteBundles } from './dialect'
|
||||
@@ -0,0 +1 @@
|
||||
console.warn('Please import @proj-airi/memory-driver-duckdb/drizzle-orm instead')
|
||||
@@ -2,10 +2,16 @@
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"lib": [
|
||||
"ESNext"
|
||||
"ESNext",
|
||||
"DOM",
|
||||
"DOM.Iterable",
|
||||
"WebWorker"
|
||||
],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"types": [
|
||||
"vite/client"
|
||||
],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
@@ -0,0 +1,7 @@
|
||||
import { defineConfig, mergeConfigs } from 'unocss'
|
||||
|
||||
import UnoCSSConfig from '../../uno.config'
|
||||
|
||||
export default defineConfig(mergeConfigs([
|
||||
UnoCSSConfig,
|
||||
]))
|
||||
@@ -0,0 +1,13 @@
|
||||
import Vue from '@vitejs/plugin-vue'
|
||||
import Unocss from 'unocss/vite'
|
||||
import { defineConfig } from 'vite'
|
||||
|
||||
export default defineConfig({
|
||||
root: 'playground',
|
||||
plugins: [
|
||||
Vue(),
|
||||
// https://github.com/antfu/unocss
|
||||
// see uno.config.ts for config
|
||||
Unocss(),
|
||||
],
|
||||
})
|
||||
@@ -1,29 +0,0 @@
|
||||
{
|
||||
"name": "@proj-airi/memory-alaya",
|
||||
"type": "module",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "",
|
||||
"author": {
|
||||
"name": "Neko Ayaka",
|
||||
"email": "neko@ayaka.moe",
|
||||
"url": "https://github.com/nekomeowww"
|
||||
},
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/moeru-ai/airi.git",
|
||||
"directory": "packages/memory-alaya"
|
||||
},
|
||||
"files": [
|
||||
"README.md",
|
||||
"dist",
|
||||
"package.json"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "pnpm run stub",
|
||||
"stub": "unbuild --stub",
|
||||
"build": "unbuild",
|
||||
"typecheck": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,6 @@ const dark = useDark()
|
||||
</template>
|
||||
|
||||
<route lang="yaml">
|
||||
meta:
|
||||
layout: default
|
||||
</route>
|
||||
meta:
|
||||
layout: default
|
||||
</route>
|
||||
|
||||
Generated
+2321
-59
File diff suppressed because it is too large
Load Diff
@@ -2,6 +2,7 @@ packages:
|
||||
- packages/**
|
||||
- services/**
|
||||
- examples/**
|
||||
- docs/**
|
||||
- '!**/dist/**'
|
||||
catalog:
|
||||
'@xsai/generate-speech': ^0.0.31
|
||||
|
||||
Reference in New Issue
Block a user