feat(stage-ui): NPR skybox environmental lighting functionality added (#457)

--------- Co-authored-by: Neko <neko@ayaka.moe>
2025-08-26 12:51:02 +08:00
parent e6b1bdc615
commit f6d6349528
8 changed files with 262 additions and 29 deletions
@@ -595,3 +595,6 @@ vrm:
  theme-color-from-model:
    button-extract:
      title: Extract
+  skybox:
+    skybox-intensity: SkyBox Intensity
+    skybox-specular-mix: Specular Mix
@@ -562,3 +562,6 @@ vrm:
  theme-color-from-model:
    button-extract:
      title: 提取
+  skybox:
+    skybox-intensity: 天空盒光照强度
+    skybox-specular-mix: 漫反射/镜面反射混合系数
@@ -37,12 +37,13 @@ const {
  ambientLightIntensity,
  ambientLightColor,

-  hemisphereLightPosition,
  hemisphereLightIntensity,
  hemisphereSkyColor,
  hemisphereGroundColor,

  envSelect,
+  skyBoxIntensity,
+  specularMix,
 } = storeToRefs(vrm)
 const trackingOptions = computed(() => [
  { value: 'camera', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.camera'), class: 'col-start-3' },
@@ -181,15 +182,6 @@ const tabList = [
          <div v-if="envSelect === 'hemisphere'">
            <!-- hemisphere settings -->
            <div grid="~ cols-5 gap-1" p-2>
-              <PropertyPoint
-                v-model:x="hemisphereLightPosition.x"
-                v-model:y="hemisphereLightPosition.y"
-                v-model:z="hemisphereLightPosition.z"
-                label="Hemisphere Light Position"
-                :x-config="{ step: 0.001, label: 'X', formatValue: val => val?.toFixed(4) }"
-                :y-config="{ step: 0.001, label: 'Y', formatValue: val => val?.toFixed(4) }"
-                :z-config="{ step: 0.001, label: 'Z', formatValue: val => val?.toFixed(4) }"
-              />
              <PropertyNumber
                v-model="hemisphereLightIntensity"
                :config="{ min: 0, max: 10, step: 0.01, label: 'Intensity' }"
@@ -207,6 +199,18 @@ const tabList = [
          </div>
          <div v-else>
            <!-- skybox settings -->
+            <div grid="~ cols-5 gap-1" p-2>
+              <PropertyNumber
+                v-model="skyBoxIntensity"
+                :config="{ min: 0, max: 1, step: 0.01, label: 'Intensity' }"
+                :label="t('settings.vrm.skybox.skybox-intensity')"
+              />
+              <PropertyNumber
+                v-model="specularMix"
+                :config="{ min: 0, max: 1, step: 0.01, label: 'Mix' }"
+                :label="t('settings.vrm.skybox.skybox-specular-mix')"
+              />
+            </div>
          </div>
        </template>
      </Tabs>
@@ -1,5 +1,7 @@
 <script setup lang="ts">
 import type { TresContext } from '@tresjs/core'
+import type { Texture,
+} from 'three'

 import { TresCanvas } from '@tresjs/core'
 import { EffectComposerPmndrs, HueSaturationPmndrs } from '@tresjs/post-processing'
@@ -7,10 +9,17 @@ import { useElementBounding, useMouse } from '@vueuse/core'
 import { formatHex } from 'culori'
 import { storeToRefs } from 'pinia'
 import { BlendFunction } from 'postprocessing'
-import { ACESFilmicToneMapping, PerspectiveCamera, Plane, Raycaster, Vector2, Vector3 } from 'three'
-import { ref, shallowRef, watch } from 'vue'
+import {
+  ACESFilmicToneMapping,
+  PerspectiveCamera,
+  Plane,
+  Raycaster,
+  Vector2,
+  Vector3,
+} from 'three'
+import { onMounted, ref, shallowRef, watch } from 'vue'

-import Environment from './VRM/Environment.vue'
+import SkyBoxEnvironment from './VRM/SkyBoxEnvironment.vue'

 import { useVRM } from '../../stores/vrm'
 import { OrbitControls, VRMModel } from '../Scenes'
@@ -52,7 +61,6 @@ const {
  ambientLightIntensity,
  ambientLightColor,

-  hemisphereLightPosition,
  hemisphereLightIntensity,
  hemisphereSkyColor,
  hemisphereGroundColor,
@@ -66,6 +74,7 @@ const modelRef = ref<InstanceType<typeof VRMModel>>()
 const camera = shallowRef(new PerspectiveCamera())
 const controlsRef = shallowRef<InstanceType<typeof OrbitControls>>()
 const tresCanvasRef = shallowRef<TresContext>()
+const skyBoxEnvRef = ref<InstanceType<typeof SkyBoxEnvironment>>()

 function onTresReady(context: TresContext) {
  tresCanvasRef.value = context
@@ -85,6 +94,9 @@ const sceneReady = ref(false)
 const raycaster = new Raycaster()
 const mouse = new Vector2()

+// For NPR skybox shader
+const nprEquirectTex = ref<Texture | null>(null)
+
 watch(cameraFOV, (newFov) => {
  if (camera.value) {
    camera.value.fov = newFov
@@ -253,6 +265,12 @@ watch(trackingMode, (newMode) => {
  }
 })

+onMounted(() => {
+  if (envSelect.value === 'skyBox') {
+    skyBoxEnvRef.value?.reload(skyBoxSrc.value)
+  }
+})
+
 defineExpose({
  setExpression: (expression: string) => {
    modelRef.value?.setExpression(expression)
@@ -277,16 +295,18 @@ defineExpose({
      @ready="onTresReady"
    >
      <OrbitControls ref="controlsRef" />
-      <Environment
+      <SkyBoxEnvironment
        v-if="envSelect === 'skyBox'"
+        ref="skyBoxEnvRef"
        :sky-box-src="skyBoxSrc"
        :as-background="true"
+        @equirect-skybox-ready="(tex) => nprEquirectTex = tex"
      />
      <TresHemisphereLight
        v-else
        :color="formatHex(hemisphereSkyColor)"
        :ground-color="formatHex(hemisphereGroundColor)"
-        :position="[hemisphereLightPosition.x, hemisphereLightPosition.y, hemisphereLightPosition.z]"
+        :position="[0, 1, 0]"
        :intensity="hemisphereLightIntensity"
        cast-shadow
      />
@@ -312,6 +332,7 @@ defineExpose({
        :model-src="props.modelSrc"
        :idle-animation="props.idleAnimation"
        :paused="props.paused"
+        :npr-equirect-tex="nprEquirectTex"
        @load-model-progress="(val) => emit('loadModelProgress', val)"
        @model-ready="handleLoadModelProgress"
        @error="(val) => emit('error', val)"
@@ -1,12 +1,27 @@
 <script setup lang="ts">
+/* eslint-disable style/max-statements-per-line */
 import type { VRMCore } from '@pixiv/three-vrm-core'
-import type { AnimationClip, Group } from 'three'
+import type { AnimationClip, Group, Texture } from 'three'

 import { VRMUtils } from '@pixiv/three-vrm'
 import { useLoop, useTresContext } from '@tresjs/core'
 import { until } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
-import { AnimationMixer, MathUtils, Mesh, MeshPhysicalMaterial, MeshStandardMaterial, Quaternion, Vector3, VectorKeyframeTrack } from 'three'
+import {
+  AnimationMixer,
+  MathUtils,
+  Mesh,
+  MeshBasicMaterial,
+  MeshPhysicalMaterial,
+  MeshStandardMaterial,
+  MeshToonMaterial,
+  Quaternion,
+  RawShaderMaterial,
+  ShaderMaterial,
+  SRGBColorSpace,
+  Vector3,
+  VectorKeyframeTrack,
+} from 'three'
 import { computed, onMounted, onUnmounted, ref, toRef, watch } from 'vue'

 import { clipFromVRMAnimation, loadVRMAnimation, useBlink, useIdleEyeSaccades } from '../../../composables/vrm/animation'
@@ -16,10 +31,10 @@ import { useVRM } from '../../../stores/vrm'

 const props = defineProps<{
  modelSrc?: string
-
  idleAnimation: string
  loadAnimations?: string[]
  paused: boolean
+  nprEquirectTex?: Texture | null
 }>()

 const emit = defineEmits<{
@@ -51,10 +66,16 @@ const {
  lookAtTarget,
  eyeHeight,
  trackingMode,
+
+  envSelect,
+  specularMix,
+  skyBoxIntensity,
 } = storeToRefs(vrmStore)
 const vrmGroup = ref<Group>()
 const idleEyeSaccades = useIdleEyeSaccades()

+const nprProgramVersion = ref(0)
+
 async function loadModel() {
  await until(modelLoading).not.toBeTruthy()
  modelLoading.value = true
@@ -190,19 +211,133 @@ async function loadModel() {

      // TODO: perhaps we should allow user to choose whether to enable
      //       this kind of behavior or not?
+      // - Lilia: NO need, cause NPR mat should always follow NPR route, and BPR mat shoule always follow PBR route.
      // WORKAROUND: set to use envMapIntensity for all matched materials
+      // - Lilia: YES, envMapIntensity will be exposed to GUI user to adjust.
      // REVIEW: MeshToonMaterial, and MeshBasicMaterial will not be affected
      //         since they do not have envMapIntensity property
+      // - Lilia: NPR mat can be affected by envMapIntensity if we choose to user skybox as envMap
+      //        This part will be implemented as a NPR HDRI shader injection
      _vrm.scene.traverse((child) => {
        if (child instanceof Mesh && child.material) {
          const material = Array.isArray(child.material) ? child.material : [child.material]
          material.forEach((mat) => {
-            // console.debug('material: ', mat)
            if (mat instanceof MeshStandardMaterial || mat instanceof MeshPhysicalMaterial) {
              // Should read envMap intensity from outside props
              mat.envMapIntensity = 1.0
              mat.needsUpdate = true
            }
+            else if (
+              mat instanceof MeshToonMaterial
+              || mat instanceof MeshBasicMaterial
+              || mat instanceof ShaderMaterial
+              || mat instanceof RawShaderMaterial
+            ) {
+              // close tone mapping for NPR materials
+              if ('toneMapped' in mat)
+                mat.toneMapped = false
+              // disable envMap for NPR materials, envMap will be reassigned by default skybox
+              if ('envMap' in mat && mat.envMap)
+                mat.envMap = null
+              // NPR materials usually use sRGB textures
+              if ('map' in mat && mat.map && 'colorSpace' in mat.map) {
+                // try...catch to avoid some weird error (diff verion of three.js?)
+                try { mat.map.colorSpace = SRGBColorSpace }
+                catch {}
+              }
+
+              // Foce recompile shader to inject npr env shader at the begining
+              const baseKey = mat.customProgramCacheKey?.() ?? ''
+              mat.customProgramCacheKey = () => `${baseKey}|npr:${nprProgramVersion.value}`
+
+              // NPR shader injection
+              const prevOnBeforeCompile = mat.onBeforeCompile
+              mat.onBeforeCompile = (shader: any, renderer: any) => {
+                // Keep the previous onBeforeCompile behaviour if any
+                prevOnBeforeCompile?.(shader, renderer)
+                // Obtain skybox texture from scene environment
+                const equirectTex = props.nprEquirectTex ?? null
+
+                // If no normal, then skip the rest
+                const hasNormal = shader.fragmentShader.includes('vNormal')
+                if (!hasNormal)
+                  return
+
+                // Setup uniforms
+                shader.uniforms.uNprEnvMode = { value: (envSelect.value === 'hemisphere') ? 0 : 2 } // 0=hemi: no further injection; 2=skybox
+                shader.uniforms.uEnvIntensity = { value: skyBoxIntensity.value } // exposed to GUI
+                shader.uniforms.uEnvMapEquirect = { value: equirectTex }
+                shader.uniforms.uSpecularMix = { value: specularMix.value }
+
+                // If it has viewDir, then we can do reflection
+                const hasView = shader.fragmentShader.includes('vViewPosition')
+
+                // Shader injection!
+                shader.fragmentShader = shader.fragmentShader.replace(
+                  '#include <common>',
+                  `
+                      #include <common>
+                      uniform int   uNprEnvMode;         // 0=off, 2=Skybox
+                      uniform float uEnvIntensity;
+                      uniform float uSpecularMix;       // 0=diffuse only, 1=specular only
+
+                      uniform sampler2D uEnvMapEquirect; // Skybox(equirect)
+                      // --- Direction to equirectangular UV ---
+                      vec2 dirToEquirectUV(vec3 d){
+                        d = normalize(d);
+                        float phi = atan(d.z, d.x);
+                        float th  = asin(clamp(d.y, -1.0, 1.0));
+                        return vec2(0.5 + phi/(2.0*PI), 0.5 - th/PI);
+                      }
+                      `,
+                ).replace(
+                  '#include <dithering_fragment>',
+                  `
+                      // --- NPR skybox env lighting injection ---
+                      vec3 n = normalize(vNormal);
+
+                      // Skybox lighting: equirect sampling -  difuse & reflection
+                      vec3 envCol = vec3(0.0);
+                      if(uNprEnvMode == 2) {
+                        ${hasView
+                          ? `vec3 v = normalize(vViewPosition);
+                            vec3 r = reflect(v, n);`
+                          : `vec3 r = n;`}
+
+                        vec3 nW = inverseTransformDirection(n, viewMatrix);
+                        vec3 rW = inverseTransformDirection(r, viewMatrix);
+                        vec3 vW = normalize(reflect(-rW, nW));
+
+                        // To resolve the upside-down reflection issue of equirect map
+                        nW.z = -nW.z;
+                        rW.z = -rW.z;
+                        
+                        float NoV = clamp(dot(nW, vW), 0.0, 1.0);
+
+                        // Specular term
+                        vec3 envSpec = texture2D(uEnvMapEquirect, dirToEquirectUV(rW)).rgb;
+                        // TODO: NPR style specular? how to do that?
+
+                        // Diffuse term
+                        vec3 envDiff    = texture2D(uEnvMapEquirect, dirToEquirectUV(-nW)).rgb;
+                        // TODO: NPR style diffuse? how to do that?
+
+                        // Mix specular and diffuse
+                        envCol = uSpecularMix * envSpec + (1.0 - uSpecularMix) * envDiff;
+
+                        // skybox color mixing 
+                        gl_FragColor.rgb += envCol * uEnvIntensity;
+                      }
+                      // --- Injection ends ---
+
+                      #include <dithering_fragment>
+                      `,
+                )
+                mat.userData.__nprUniforms = shader.uniforms
+              }
+              mat.needsUpdate = true
+            }
+            // console.debug('material: ', mat)
          })
        }
      })
@@ -285,6 +420,39 @@ function componentCleanUp() {
  }
 }

+// Switch to NPR SkyBox
+function updateNprUniforms(tex: Texture | null) {
+  const root = vrm.value?.scene
+  if (!root)
+    return
+
+  const mode = (envSelect.value === 'skyBox' && !!tex) ? 2 : 0 // 0=off，2=skybox
+  root.traverse((child) => {
+    if (child instanceof Mesh && child.material) {
+      const mats = Array.isArray(child.material) ? child.material : [child.material]
+      mats.forEach((mat) => {
+        const u = mat.userData?.__nprUniforms
+        if (!u)
+          return
+        // Skybox texture and mode
+        u.uEnvMapEquirect.value = tex
+        u.uNprEnvMode.value = mode
+        u.uEnvIntensity.value = skyBoxIntensity.value
+        u.uSpecularMix.value = specularMix.value
+      })
+    }
+  })
+}
+// watch NPR skybox
+watch(
+  () => [envSelect.value, props.nprEquirectTex, skyBoxIntensity.value, specularMix.value],
+  async () => {
+    nprProgramVersion.value += 1
+    updateNprUniforms(props.nprEquirectTex ?? null)
+  },
+  { immediate: true, deep: false },
+)
+
 onMounted(async () => await loadModel())
 onUnmounted(() => componentCleanUp())

@@ -1,9 +1,18 @@
 <script setup lang="ts">
-import type { CanvasTexture, DataTexture, Scene, WebGLRenderer, WebGLRenderTarget } from 'three'
+import type {
+  CanvasTexture,
+  DataTexture,
+  Scene,
+  Texture,
+  WebGLRenderer,
+  WebGLRenderTarget,
+} from 'three'

 import { useTresContext } from '@tresjs/core'
+import { until } from '@vueuse/core'
 import {
  ACESFilmicToneMapping,
+  EquirectangularReflectionMapping,
  PMREMGenerator,
  SRGBColorSpace,
 } from 'three'
@@ -30,10 +39,16 @@ const props = withDefaults(defineProps<{
  backgroundIntensity: 1,
 })

+// emit equirect HDRI for NPR shader use
+const emit = defineEmits<{
+  (e: 'equirectSkyboxReady', value: Texture | null): void
+}>()
+
 // Add these reactive variables to your setup
 const environment = ref<DataTexture | CanvasTexture>()
 let _pmrem: PMREMGenerator | null = null
 let _envRT: WebGLRenderTarget | null = null // WebGLRenderTarget from PMREM
+let _equirectTex: Texture | null = null // equirectangular texture for NPR shader

 const { scene, renderer } = useTresContext()

@@ -46,16 +61,24 @@ function clearEnvironment() {
  scn.background = null
  // Do not forcibly clear background; caller/prop controls that intent.
  // scn.background = null
+  // Lilia: background must be cleared when switching to hemisphere light
  _envRT?.dispose?.()
  _pmrem?.dispose?.()
  _envRT = null
  _pmrem = null
+
+  // Clear equirect texture for NPR skybox
+  // if(_equirectTex) {
+  //   emit('equirect-skybox-ready', null)
+  //   // _equirectTex.dispose?.()
+  //   // _equirectTex = null
+  // }
 }

 // load HDRI environment from sky box
 async function loadEnvironment(skyBoxSrc?: string | null) {
-  if (!scene.value || !renderer.value)
-    return
+  // Wait until renderer is ready
+  await until(() => !!renderer.value && !!renderer.value).toBeTruthy()

  // Always dispose previous env when switching
  clearEnvironment()
@@ -75,6 +98,7 @@ async function loadEnvironment(skyBoxSrc?: string | null) {
    // - polyhaven.com
    // - hdrihaven.com
    const hdrTex = await new RGBELoader().loadAsync(skyBoxSrc)
+    hdrTex.mapping = EquirectangularReflectionMapping

    // PMREM prefiltering for physically correct IBL
    _pmrem = new PMREMGenerator(renderer.value as WebGLRenderer)
@@ -90,8 +114,10 @@ async function loadEnvironment(skyBoxSrc?: string | null) {
    scn.backgroundBlurriness = props.backgroundBlurriness
    scn.backgroundIntensity = props.backgroundIntensity

-    // Source HDR no longer needed for sampling by materials
-    hdrTex.dispose()
+    // emit equirect texture for NPR shader use
+    // Don't dispose this texture, it will be used by NPR injected shader
+    _equirectTex = hdrTex
+    emit('equirectSkyboxReady', _equirectTex)
  }
  catch (error) {
    console.warn('Failed to load HDRI environment:', error)
@@ -115,6 +141,10 @@ onMounted(async () => {
  )
 })

+defineExpose({
+  reload: async (skyBoxSrc: string | null) => await loadEnvironment(skyBoxSrc),
+})
+
 onUnmounted(async () => {
  await clearEnvironment()
 })
@@ -1,2 +1,3 @@
 export { default as VRMModel } from './Model.vue'
 export { default as OrbitControls } from './OrbitControls.vue'
+export { default as SkyBoxEnvironment } from './SkyBoxEnvironment.vue'
@@ -59,11 +59,11 @@ export const useVRM = defineStore('vrm', () => {
  // TODO: color are the same
  const directionalLightColor = useLocalStorage('settings/vrm/scenes/scene/directional-light/color', '#fffbf5')

-  const hemisphereLightPosition = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/position', { x: 0, y: 0, z: 0 })
  // TODO: color are the same
  const hemisphereSkyColor = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/sky-color', '#FFFFFF')
  // TODO: color are the same
-  const hemisphereGroundColor = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/ground-color', '#000000')
+  // Lilia: The ground color is not pure black
+  const hemisphereGroundColor = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/ground-color', '#222222')
  // TODO: The same as directional light, this is a temporary solution
  //       and will be replaced with a more flexible lighting system in the future.
  const hemisphereLightIntensity = useLocalStorage('settings/vrm/scenes/scene/hemisphere-light/intensity', 0.4)
@@ -80,8 +80,10 @@ export const useVRM = defineStore('vrm', () => {
  const eyeHeight = useLocalStorage('settings/vrm/eyeHeight', 0)

  // environment related setting
-  const envSelect = useLocalStorage('settings/vrm/envEnabled', 'hemisphere' as 'hemisphere' | 'skyBox')
+  const envSelect = useLocalStorage('settings/vrm/envEnabled', 'skyBox' as 'hemisphere' | 'skyBox')
  const skyBoxSrc = useLocalStorage('settings/vrm/skyBoxUrl', defaultSkyBoxSrc)
+  const specularMix = useLocalStorage('settings/vrm/specularMix', 0)
+  const skyBoxIntensity = useLocalStorage('settings/vrm/skyBoxIntensity', 0.1)

  return {
    modelSize,
@@ -104,7 +106,6 @@ export const useVRM = defineStore('vrm', () => {
    ambientLightIntensity,
    ambientLightColor,

-    hemisphereLightPosition,
    hemisphereSkyColor,
    hemisphereGroundColor,
    hemisphereLightIntensity,
@@ -115,6 +116,8 @@ export const useVRM = defineStore('vrm', () => {
    eyeHeight,
    envSelect,
    skyBoxSrc,
+    specularMix,
+    skyBoxIntensity,

    shouldUpdateView,
    onShouldUpdateView,