nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(stage-ui): NPR skybox environmental lighting functionality added (#457)

Browse Source 
---------

Co-authored-by: Neko <neko@ayaka.moe>
This commit is contained in:
Lilia_Chen
2025-08-26 12:51:02 +08:00
committed by GitHub
co-authored by Neko
parent e6b1bdc615
commit f6d6349528
8 changed files with 262 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/i18n/src/locales/en/settings.yaml
+3
View File
@@ -595,3 +595,6 @@ vrm:
theme-color-from-model:
button-extract:
title: Extract
skybox:
skybox-intensity: SkyBox Intensity
skybox-specular-mix: Specular Mix
packages/i18n/src/locales/zh-Hans/settings.yaml
+3
View File
@@ -562,3 +562,6 @@ vrm:
theme-color-from-model:
button-extract:
title: 提取
skybox:
skybox-intensity: 天空盒光照强度
skybox-specular-mix: 漫反射/镜面反射混合系数
packages/stage-ui/src/components/Scenarios/Settings/ModelSettings/VRM.vue
+14 -10
View File
@@ -37,12 +37,13 @@ const {
ambientLightIntensity,
ambientLightColor,
hemisphereLightPosition,
hemisphereLightIntensity,
hemisphereSkyColor,
hemisphereGroundColor,
envSelect,
skyBoxIntensity,
specularMix,
} = storeToRefs(vrm)
const trackingOptions = computed(() => [
{ value: 'camera', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.camera'), class: 'col-start-3' },
@@ -181,15 +182,6 @@ const tabList = [
<div v-if="envSelect === 'hemisphere'">
<!-- hemisphere settings -->
<div grid="~ cols-5 gap-1" p-2>
<PropertyPoint
v-model:x="hemisphereLightPosition.x"
v-model:y="hemisphereLightPosition.y"
v-model:z="hemisphereLightPosition.z"
label="Hemisphere Light Position"
:x-config="{ step: 0.001, label: 'X', formatValue: val => val?.toFixed(4) }"
:y-config="{ step: 0.001, label: 'Y', formatValue: val => val?.toFixed(4) }"
:z-config="{ step: 0.001, label: 'Z', formatValue: val => val?.toFixed(4) }"
/>
<PropertyNumber
v-model="hemisphereLightIntensity"
:config="{ min: 0, max: 10, step: 0.01, label: 'Intensity' }"
@@ -207,6 +199,18 @@ const tabList = [
</div>
<div v-else>
<!-- skybox settings -->
<div grid="~ cols-5 gap-1" p-2>
<PropertyNumber
v-model="skyBoxIntensity"
:config="{ min: 0, max: 1, step: 0.01, label: 'Intensity' }"
:label="t('settings.vrm.skybox.skybox-intensity')"
/>
<PropertyNumber
v-model="specularMix"
:config="{ min: 0, max: 1, step: 0.01, label: 'Mix' }"
:label="t('settings.vrm.skybox.skybox-specular-mix')"
/>
</div>
</div>
</template>
</Tabs>
packages/stage-ui/src/components/Scenes/VRM.vue
+27 -6
View File
@@ -1,5 +1,7 @@
<script setup lang="ts">
import type { TresContext } from '@tresjs/core'
import type { Texture,
} from 'three'
import { TresCanvas } from '@tresjs/core'
import { EffectComposerPmndrs, HueSaturationPmndrs } from '@tresjs/post-processing'
@@ -7,10 +9,17 @@ import { useElementBounding, useMouse } from '@vueuse/core'
import { formatHex } from 'culori'
import { storeToRefs } from 'pinia'
import { BlendFunction } from 'postprocessing'
import { ACESFilmicToneMapping, PerspectiveCamera, Plane, Raycaster, Vector2, Vector3 } from 'three'
import { ref, shallowRef, watch } from 'vue'
import {
ACESFilmicToneMapping,
PerspectiveCamera,
Plane,
Raycaster,
Vector2,
Vector3,
} from 'three'
import { onMounted, ref, shallowRef, watch } from 'vue'
import Environment from './VRM/Environment.vue'
import SkyBoxEnvironment from './VRM/SkyBoxEnvironment.vue'
import { useVRM } from '../../stores/vrm'
import { OrbitControls, VRMModel } from '../Scenes'
@@ -52,7 +61,6 @@ const {
ambientLightIntensity,
ambientLightColor,
hemisphereLightPosition,
hemisphereLightIntensity,
hemisphereSkyColor,
hemisphereGroundColor,
@@ -66,6 +74,7 @@ const modelRef = ref<InstanceType<typeof VRMModel>>()
const camera = shallowRef(new PerspectiveCamera())
const controlsRef = shallowRef<InstanceType<typeof OrbitControls>>()
const tresCanvasRef = shallowRef<TresContext>()
const skyBoxEnvRef = ref<InstanceType<typeof SkyBoxEnvironment>>()
function onTresReady(context: TresContext) {
tresCanvasRef.value = context
@@ -85,6 +94,9 @@ const sceneReady = ref(false)
const raycaster = new Raycaster()
const mouse = new Vector2()
// For NPR skybox shader
const nprEquirectTex = ref<Texture | null>(null)
watch(cameraFOV, (newFov) => {
if (camera.value) {
camera.value.fov = newFov
@@ -253,6 +265,12 @@ watch(trackingMode, (newMode) => {
}
})
onMounted(() => {
if (envSelect.value === 'skyBox') {
skyBoxEnvRef.value?.reload(skyBoxSrc.value)
}
})
defineExpose({
setExpression: (expression: string) => {
modelRef.value?.setExpression(expression)
@@ -277,16 +295,18 @@ defineExpose({
@ready="onTresReady"
>
<OrbitControls ref="controlsRef" />
<Environment
<SkyBoxEnvironment
v-if="envSelect === 'skyBox'"
ref="skyBoxEnvRef"
:sky-box-src="skyBoxSrc"
:as-background="true"
@equirect-skybox-ready="(tex) => nprEquirectTex = tex"
/>
<TresHemisphereLight
v-else
:color="formatHex(hemisphereSkyColor)"
:ground-color="formatHex(hemisphereGroundColor)"
:position="[hemisphereLightPosition.x, hemisphereLightPosition.y, hemisphereLightPosition.z]"
:position="[0, 1, 0]"
:intensity="hemisphereLightIntensity"
cast-shadow
/>
@@ -312,6 +332,7 @@ defineExpose({
:model-src="props.modelSrc"
:idle-animation="props.idleAnimation"
:paused="props.paused"
:npr-equirect-tex="nprEquirectTex"
@load-model-progress="(val) => emit('loadModelProgress', val)"
@model-ready="handleLoadModelProgress"
@error="(val) => emit('error', val)"
packages/stage-ui/src/components/Scenes/VRM/Model.vue
+172 -4
View File
@@ -1,12 +1,27 @@
<script setup lang="ts">
/* eslint-disable style/max-statements-per-line */
import type { VRMCore } from '@pixiv/three-vrm-core'
import type { AnimationClip, Group } from 'three'
import type { AnimationClip, Group, Texture } from 'three'
import { VRMUtils } from '@pixiv/three-vrm'
import { useLoop, useTresContext } from '@tresjs/core'
import { until } from '@vueuse/core'
import { storeToRefs } from 'pinia'
import { AnimationMixer, MathUtils, Mesh, MeshPhysicalMaterial, MeshStandardMaterial, Quaternion, Vector3, VectorKeyframeTrack } from 'three'
import {
AnimationMixer,
MathUtils,
Mesh,
MeshBasicMaterial,
MeshPhysicalMaterial,
MeshStandardMaterial,
MeshToonMaterial,
Quaternion,
RawShaderMaterial,
ShaderMaterial,
SRGBColorSpace,
Vector3,
VectorKeyframeTrack,
} from 'three'
import { computed, onMounted, onUnmounted, ref, toRef, watch } from 'vue'
import { clipFromVRMAnimation, loadVRMAnimation, useBlink, useIdleEyeSaccades } from '../../../composables/vrm/animation'
@@ -16,10 +31,10 @@ import { useVRM } from '../../../stores/vrm'
const props = defineProps<{
modelSrc?: string
idleAnimation: string
loadAnimations?: string[]
paused: boolean
nprEquirectTex?: Texture | null
}>()
const emit = defineEmits<{
@@ -51,10 +66,16 @@ const {
lookAtTarget,
eyeHeight,
trackingMode,
envSelect,
specularMix,
skyBoxIntensity,
} = storeToRefs(vrmStore)
const vrmGroup = ref<Group>()
const idleEyeSaccades = useIdleEyeSaccades()
const nprProgramVersion = ref(0)
async function loadModel() {
await until(modelLoading).not.toBeTruthy()
modelLoading.value = true
@@ -190,19 +211,133 @@ async function loadModel() {
// TODO: perhaps we should allow user to choose whether to enable
// this kind of behavior or not?
// - Lilia: NO need, cause NPR mat should always follow NPR route, and BPR mat shoule always follow PBR route.
// WORKAROUND: set to use envMapIntensity for all matched materials
// - Lilia: YES, envMapIntensity will be exposed to GUI user to adjust.
// REVIEW: MeshToonMaterial, and MeshBasicMaterial will not be affected
// since they do not have envMapIntensity property
// - Lilia: NPR mat can be affected by envMapIntensity if we choose to user skybox as envMap
// This part will be implemented as a NPR HDRI shader injection
_vrm.scene.traverse((child) => {
if (child instanceof Mesh && child.material) {
const material = Array.isArray(child.material) ? child.material : [child.material]
material.forEach((mat) => {
// console.debug('material: ', mat)
if (mat instanceof MeshStandardMaterial || mat instanceof MeshPhysicalMaterial) {
// Should read envMap intensity from outside props
mat.envMapIntensity = 1.0
mat.needsUpdate = true
}
else if (
mat instanceof MeshToonMaterial
|| mat instanceof MeshBasicMaterial
|| mat instanceof ShaderMaterial
|| mat instanceof RawShaderMaterial
) {
// close tone mapping for NPR materials
if ('toneMapped' in mat)
mat.toneMapped = false
// disable envMap for NPR materials, envMap will be reassigned by default skybox
if ('envMap' in mat && mat.envMap)
mat.envMap = null
// NPR materials usually use sRGB textures
if ('map' in mat && mat.map && 'colorSpace' in mat.map) {
// try...catch to avoid some weird error (diff verion of three.js?)
try { mat.map.colorSpace = SRGBColorSpace }
catch {}
}
// Foce recompile shader to inject npr env shader at the begining
const baseKey = mat.customProgramCacheKey?.() ?? ''
mat.customProgramCacheKey = () => `${baseKey}|npr:${nprProgramVersion.value}`
// NPR shader injection
const prevOnBeforeCompile = mat.onBeforeCompile
mat.onBeforeCompile = (shader: any, renderer: any) => {
// Keep the previous onBeforeCompile behaviour if any
prevOnBeforeCompile?.(shader, renderer)
// Obtain skybox texture from scene environment
const equirectTex = props.nprEquirectTex ?? null
// If no normal, then skip the rest
const hasNormal = shader.fragmentShader.includes('vNormal')
if (!hasNormal)
return
// Setup uniforms
shader.uniforms.uNprEnvMode = { value: (envSelect.value === 'hemisphere') ? 0 : 2 } // 0=hemi: no further injection; 2=skybox
shader.uniforms.uEnvIntensity = { value: skyBoxIntensity.value } // exposed to GUI
shader.uniforms.uEnvMapEquirect = { value: equirectTex }
shader.uniforms.uSpecularMix = { value: specularMix.value }
// If it has viewDir, then we can do reflection
const hasView = shader.fragmentShader.includes('vViewPosition')
// Shader injection!
shader.fragmentShader = shader.fragmentShader.replace(
'#include <common>',
`
#include <common>
uniform int uNprEnvMode; // 0=off, 2=Skybox
uniform float uEnvIntensity;
uniform float uSpecularMix; // 0=diffuse only, 1=specular only
uniform sampler2D uEnvMapEquirect; // Skybox(equirect)
// --- Direction to equirectangular UV ---
vec2 dirToEquirectUV(vec3 d){
d = normalize(d);
float phi = atan(d.z, d.x);
float th = asin(clamp(d.y, -1.0, 1.0));
return vec2(0.5 + phi/(2.0*PI), 0.5 - th/PI);
}
`,
).replace(
'#include <dithering_fragment>',
`
// --- NPR skybox env lighting injection ---
vec3 n = normalize(vNormal);
// Skybox lighting: equirect sampling - difuse & reflection
vec3 envCol = vec3(0.0);
if(uNprEnvMode == 2) {
${hasView
? `vec3 v = normalize(vViewPosition);
vec3 r = reflect(v, n);`
: `vec3 r = n;`}
vec3 nW = inverseTransformDirection(n, viewMatrix);
vec3 rW = inverseTransformDirection(r, viewMatrix);
vec3 vW = normalize(reflect(-rW, nW));
// To resolve the upside-down reflection issue of equirect map
nW.z = -nW.z;
rW.z = -rW.z;
float NoV = clamp(dot(nW, vW), 0.0, 1.0);
// Specular term
vec3 envSpec = texture2D(uEnvMapEquirect, dirToEquirectUV(rW)).rgb;
// TODO: NPR style specular? how to do that?
// Diffuse term
vec3 envDiff = texture2D(uEnvMapEquirect, dirToEquirectUV(-nW)).rgb;
// TODO: NPR style diffuse? how to do that?
// Mix specular and diffuse
envCol = uSpecularMix * envSpec + (1.0 - uSpecularMix) * envDiff;
// skybox color mixing
gl_FragColor.rgb += envCol * uEnvIntensity;
}
// --- Injection ends ---
#include <dithering_fragment>
`,
)
mat.userData.__nprUniforms = shader.uniforms
}
mat.needsUpdate = true
}
// console.debug('material: ', mat)
})
}
})
@@ -285,6 +420,39 @@ function componentCleanUp() {
}
}
// Switch to NPR SkyBox
function updateNprUniforms(tex: Texture | null) {
const root = vrm.value?.scene
if (!root)
return
const mode = (envSelect.value === 'skyBox' && !!tex) ? 2 : 0 // 0=off，2=skybox
root.traverse((child) => {
if (child instanceof Mesh && child.material) {
const mats = Array.isArray(child.material) ? child.material : [child.material]
mats.forEach((mat) => {
const u = mat.userData?.__nprUniforms
if (!u)
return
// Skybox texture and mode
u.uEnvMapEquirect.value = tex
u.uNprEnvMode.value = mode
u.uEnvIntensity.value = skyBoxIntensity.value
u.uSpecularMix.value = specularMix.value
})
}
})
}
// watch NPR skybox
watch(
() => [envSelect.value, props.nprEquirectTex, skyBoxIntensity.value, specularMix.value],
async () => {
nprProgramVersion.value += 1
updateNprUniforms(props.nprEquirectTex ?? null)
},
{ immediate: true, deep: false },
)
onMounted(async () => await loadModel())
onUnmounted(() => componentCleanUp())
packages/stage-ui/src/components/Scenes/VRM/Environment.vue → packages/stage-ui/src/components/Scenes/VRM/SkyBoxEnvironment.vue
+35 -5
View File
@@ -1,9 +1,18 @@
<script setup lang="ts">
import type { CanvasTexture, DataTexture, Scene, WebGLRenderer, WebGLRenderTarget } from 'three'
import type {
CanvasTexture,
DataTexture,
Scene,
Texture,
WebGLRenderer,
WebGLRenderTarget,
} from 'three'
import { useTresContext } from '@tresjs/core'
import { until } from '@vueuse/core'
import {
ACESFilmicToneMapping,
EquirectangularReflectionMapping,
PMREMGenerator,
SRGBColorSpace,
} from 'three'
@@ -30,10 +39,16 @@ const props = withDefaults(defineProps<{
backgroundIntensity: 1,
})
// emit equirect HDRI for NPR shader use
const emit = defineEmits<{
(e: 'equirectSkyboxReady', value: Texture | null): void
}>()
// Add these reactive variables to your setup
const environment = ref<DataTexture | CanvasTexture>()
let _pmrem: PMREMGenerator | null = null
let _envRT: WebGLRenderTarget | null = null // WebGLRenderTarget from PMREM
let _equirectTex: Texture | null = null // equirectangular texture for NPR shader
const { scene, renderer } = useTresContext()
@@ -46,16 +61,24 @@ function clearEnvironment() {
scn.background = null
// Do not forcibly clear background; caller/prop controls that intent.
// scn.background = null
// Lilia: background must be cleared when switching to hemisphere light
_envRT?.dispose?.()
_pmrem?.dispose?.()
_envRT = null
_pmrem = null
// Clear equirect texture for NPR skybox
// if(_equirectTex) {
// emit('equirect-skybox-ready', null)
// // _equirectTex.dispose?.()
// // _equirectTex = null
// }
}
// load HDRI environment from sky box
async function loadEnvironment(skyBoxSrc?: string | null) {
if (!scene.value || !renderer.value)
return
// Wait until renderer is ready
await until(() => !!renderer.value && !!renderer.value).toBeTruthy()
// Always dispose previous env when switching
clearEnvironment()
@@ -75,6 +98,7 @@ async function loadEnvironment(skyBoxSrc?: string | null) {
// - polyhaven.com
// - hdrihaven.com
const hdrTex = await new RGBELoader().loadAsync(skyBoxSrc)
hdrTex.mapping = EquirectangularReflectionMapping
// PMREM prefiltering for physically correct IBL
_pmrem = new PMREMGenerator(renderer.value as WebGLRenderer)
@@ -90,8 +114,10 @@ async function loadEnvironment(skyBoxSrc?: string | null) {
scn.backgroundBlurriness = props.backgroundBlurriness
scn.backgroundIntensity = props.backgroundIntensity
// Source HDR no longer needed for sampling by materials
hdrTex.dispose()
// emit equirect texture for NPR shader use
// Don't dispose this texture, it will be used by NPR injected shader
_equirectTex = hdrTex
emit('equirectSkyboxReady', _equirectTex)
}
catch (error) {
console.warn('Failed to load HDRI environment:', error)
@@ -115,6 +141,10 @@ onMounted(async () => {
)
})
defineExpose({
reload: async (skyBoxSrc: string | null) => await loadEnvironment(skyBoxSrc),
})
onUnmounted(async () => {
await clearEnvironment()
})
packages/stage-ui/src/components/Scenes/VRM/index.ts
+1
View File
@@ -1,2 +1,3 @@
export { default as VRMModel } from './Model.vue'
export { default as OrbitControls } from './OrbitControls.vue'
export { default as SkyBoxEnvironment } from './SkyBoxEnvironment.vue'
packages/stage-ui/src/stores/vrm.ts
+7 -4
View File
@@ -59,11 +59,11 @@ export const useVRM = defineStore('vrm', () => {
// TODO: color are the same
const directionalLightColor = useLocalStorage('settings/vrm/scenes/scene/directional-light/color', '#fffbf5')
const hemisphereLightPosition = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/position', { x: 0, y: 0, z: 0 })
// TODO: color are the same
const hemisphereSkyColor = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/sky-color', '#FFFFFF')
// TODO: color are the same
const hemisphereGroundColor = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/ground-color', '#000000')
// Lilia: The ground color is not pure black
const hemisphereGroundColor = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/ground-color', '#222222')
// TODO: The same as directional light, this is a temporary solution
// and will be replaced with a more flexible lighting system in the future.
const hemisphereLightIntensity = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/intensity', 0.4)
@@ -80,8 +80,10 @@ export const useVRM = defineStore('vrm', () => {
const eyeHeight = useLocalStorage('settings/vrm/eyeHeight', 0)
// environment related setting
const envSelect = useLocalStorage('settings/vrm/envEnabled', 'hemisphere' as 'hemisphere' | 'skyBox')
const envSelect = useLocalStorage('settings/vrm/envEnabled', 'skyBox' as 'hemisphere' | 'skyBox')
const skyBoxSrc = useLocalStorage('settings/vrm/skyBoxUrl', defaultSkyBoxSrc)
const specularMix = useLocalStorage('settings/vrm/specularMix', 0)
const skyBoxIntensity = useLocalStorage('settings/vrm/skyBoxIntensity', 0.1)
return {
modelSize,
@@ -104,7 +106,6 @@ export const useVRM = defineStore('vrm', () => {
ambientLightIntensity,
ambientLightColor,
hemisphereLightPosition,
hemisphereSkyColor,
hemisphereGroundColor,
hemisphereLightIntensity,
@@ -115,6 +116,8 @@ export const useVRM = defineStore('vrm', () => {
eyeHeight,
envSelect,
skyBoxSrc,
specularMix,
skyBoxIntensity,
shouldUpdateView,
onShouldUpdateView,