maobo/cup_edit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TextureProjection now makes a separate render pass for each View.

Browse Source 
This is needed because changing material properties between beginFrame/endFrame is unsupported
This commit is contained in:
Nick Fisher
2025-05-22 15:03:06 +08:00
parent 1008452198
commit 3007dec127
2 changed files with 118 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
thermion_dart/lib/src/utils/src/texture_projection.dart
View File

@@ -1,35 +1,37 @@
import 'dart:io'; import 'dart:io';
import 'dart:typed_data';
import 'package:thermion_dart/src/filament/src/implementation/ffi_asset.dart';
import 'package:thermion_dart/src/filament/src/implementation/ffi_material.dart';
import 'package:thermion_dart/src/filament/src/implementation/ffi_camera.dart';
import 'package:thermion_dart/src/filament/src/implementation/ffi_render_target.dart'; import 'package:thermion_dart/src/filament/src/implementation/ffi_render_target.dart';
import 'package:thermion_dart/src/filament/src/implementation/ffi_scene.dart'; import 'package:thermion_dart/src/filament/src/implementation/ffi_scene.dart';
import 'package:thermion_dart/src/filament/src/implementation/ffi_view.dart'; import 'package:thermion_dart/src/filament/src/implementation/ffi_view.dart';
import 'package:thermion_dart/src/filament/src/implementation/ffi_texture.dart';
import 'package:thermion_dart/thermion_dart.dart'; import 'package:thermion_dart/thermion_dart.dart';
class TextureProjection { class TextureProjection {
final SwapChain swapChain;
final Material projectionMaterial; final Material projectionMaterial;
final MaterialInstance projectionMaterialInstance; final FFIMaterialInstance projectionMaterialInstance;
final Material depthWriteMaterial; final Material depthWriteMaterial;
final MaterialInstance depthWriteMaterialInstance; final FFIMaterialInstance depthWriteMaterialInstance;
final Texture texture; final FFIView sourceView;
final View sourceView; final FFIView depthView;
final View depthView; final FFIView projectionView;
final View projectionView; final Texture depthWriteColorTexture;
final FFITextureSampler sampler;
TextureProjection._( TextureProjection._(
{required this.swapChain, {required this.projectionMaterial,
required this.projectionMaterial,
required this.projectionMaterialInstance, required this.projectionMaterialInstance,
required this.depthWriteMaterial, required this.depthWriteMaterial,
required this.depthWriteMaterialInstance, required this.depthWriteMaterialInstance,
required this.texture,
required this.sourceView, required this.sourceView,
required this.depthView, required this.depthView,
required this.projectionView}) {} required this.projectionView,
required this.depthWriteColorTexture,
required this.sampler}) {}
static Future<TextureProjection> create( static Future<TextureProjection> create(View sourceView) async {
View sourceView, SwapChain swapChain) async {
final viewport = await sourceView.getViewport(); final viewport = await sourceView.getViewport();
var depthWriteMat = await FilamentApp.instance!.createMaterial( var depthWriteMat = await FilamentApp.instance!.createMaterial(
File( File(
@@ -38,11 +40,11 @@ class TextureProjection {
); );
var depthWriteMi = await depthWriteMat.createInstance(); var depthWriteMi = await depthWriteMat.createInstance();
final depthWriteView = await FilamentApp.instance!.createView() as FFIView; final depthView = await FilamentApp.instance!.createView() as FFIView;
await depthWriteView.setFrustumCullingEnabled(false); await depthView.setRenderable(true);
await depthWriteView.setPostProcessing(false); await depthView.setFrustumCullingEnabled(false);
await depthWriteView.setViewport(viewport.width, viewport.height); await depthView.setPostProcessing(false);
await depthWriteView.setBlendMode(BlendMode.transparent); await depthView.setViewport(viewport.width, viewport.height);
final depthWriteColorTexture = await FilamentApp.instance! final depthWriteColorTexture = await FilamentApp.instance!
.createTexture(viewport.width, viewport.height, .createTexture(viewport.width, viewport.height,
@@ -51,100 +53,137 @@ class TextureProjection {
TextureUsage.TEXTURE_USAGE_SAMPLEABLE, TextureUsage.TEXTURE_USAGE_SAMPLEABLE,
TextureUsage.TEXTURE_USAGE_BLIT_SRC TextureUsage.TEXTURE_USAGE_BLIT_SRC
}, },
textureFormat: TextureFormat.R32F); textureFormat: TextureFormat.R32F) as FFITexture;
await depthWriteView final depthWriteRenderTarget =
.setRenderTarget(await FilamentApp.instance!.createRenderTarget( await FilamentApp.instance!.createRenderTarget(
viewport.width, viewport.width,
viewport.height, viewport.height,
color: depthWriteColorTexture, color: depthWriteColorTexture,
) as FFIRenderTarget); ) as FFIRenderTarget;
await depthView.setRenderTarget(depthWriteRenderTarget);
final color = await (await sourceView.getRenderTarget())!.getColorTexture(); final captureMat = await FilamentApp.instance!.createMaterial(
final depth =
await (await depthWriteView.getRenderTarget())!.getColorTexture();
var captureMat = await FilamentApp.instance!.createMaterial(
File( File(
"/Users/nickfisher/Documents/thermion/materials/capture_uv.filamat", "/Users/nickfisher/Documents/thermion/materials/capture_uv.filamat",
).readAsBytesSync(), ).readAsBytesSync(),
); ) as FFIMaterial;
var captureMi = await captureMat.createInstance(); var captureMi = await captureMat.createInstance() as FFIMaterialInstance;
await captureMi.setParameterBool("flipUVs", true); await captureMi.setParameterBool("flipUVs", true);
final sampler =
await FilamentApp.instance!.createTextureSampler() as FFITextureSampler;
await captureMi.setParameterTexture( await captureMi.setParameterTexture(
"color", color, await FilamentApp.instance!.createTextureSampler()); "depth", depthWriteColorTexture, sampler);
await captureMi.setParameterTexture(
"depth", depth, await FilamentApp.instance!.createTextureSampler());
await captureMi.setParameterBool("useDepth", true); await captureMi.setParameterBool("useDepth", true);
final projectionView = await FilamentApp.instance!.createView() as FFIView; final projectionView = await FilamentApp.instance!.createView() as FFIView;
final projectionRenderTarget = await FilamentApp.instance!
.createRenderTarget(viewport.width, viewport.height) as FFIRenderTarget;
await projectionView.setFrustumCullingEnabled(false); await projectionView.setFrustumCullingEnabled(false);
await projectionView.setPostProcessing(false); await projectionView.setPostProcessing(false);
await projectionView.setViewport(viewport.width, viewport.height); await projectionView.setViewport(viewport.width, viewport.height);
await projectionView.setRenderTarget(projectionRenderTarget);
final swapChain = await FilamentApp.instance!.getSwapChain(sourceView);
await FilamentApp.instance!.register(swapChain, depthView);
await FilamentApp.instance!.register(swapChain, projectionView);
return TextureProjection._( return TextureProjection._(
sourceView: sourceView, sourceView: sourceView as FFIView,
swapChain: swapChain, depthView: depthView,
depthView: depthWriteView,
projectionView: projectionView, projectionView: projectionView,
projectionMaterial: captureMat, projectionMaterial: captureMat,
projectionMaterialInstance: captureMi, projectionMaterialInstance: captureMi,
depthWriteMaterial: depthWriteMat, depthWriteMaterial: depthWriteMat,
depthWriteMaterialInstance: depthWriteMi, depthWriteMaterialInstance: depthWriteMi as FFIMaterialInstance,
texture: depthWriteColorTexture); depthWriteColorTexture: depthWriteColorTexture,
sampler: sampler);
} }
Future destroy() async { Future destroy() async {
final swapChain = await FilamentApp.instance!.getSwapChain(sourceView);
await FilamentApp.instance!.unregister(swapChain, depthView);
await FilamentApp.instance!.unregister(swapChain, projectionView);
await projectionMaterialInstance.destroy(); await projectionMaterialInstance.destroy();
await projectionMaterial.destroy(); await projectionMaterial.destroy();
await FilamentApp.instance!.destroyView(depthView); await FilamentApp.instance!.destroyView(depthView);
await FilamentApp.instance!.destroyView(projectionView); await FilamentApp.instance!.destroyView(projectionView);
} }
var _pixelBuffers = <View, Uint8List>{}; /// Projects/unwraps [texture] onto [target] based on the current view/camera
Uint8List getColorBuffer() => _pixelBuffers[sourceView]!; /// and the UV coordinates for [target].
Uint8List getDepthWritePixelBuffer() => _pixelBuffers[depthView]!; ///
Uint8List getProjectedPixelBuffer() => _pixelBuffers[projectionView]!; /// 1) Create a new scene only containing the target asset
/// 2) Assign a material to the target asset that writes the depth of each
/// fragment to an output texture
/// 3) Render this "depth view" to a render target
/// 4) Assign a material to the target asset that:
/// a) transforms each vertex position to its UV coordinates
/// b) colors each fragment blue
/// 5) Use the render target color buffer as the input to a
/// 6) Render this "projection view" and capture the output
Future<TextureProjectionResult> project(Texture texture, ThermionAsset target,
{bool renderSourceView = true}) async {
await FilamentApp.instance!.setClearOptions(0, 0, 0, 1,
clearStencil: 0, discard: false, clear: true);
final viewport = await sourceView.getViewport();
Future project(ThermionAsset target) async {
final originalMi = await target.getMaterialInstanceAt(); final originalMi = await target.getMaterialInstanceAt();
await FilamentApp.instance!.register(swapChain, depthView); final camera = (await sourceView.getCamera()) as FFICamera;
await FilamentApp.instance!.register(swapChain, projectionView); final originalScene = await sourceView.getScene() as FFIScene;
final projectionScene =
(await FilamentApp.instance!.createScene()) as FFIScene;
await projectionScene.add(target as FFIAsset);
await sourceView.setScene(projectionScene);
final camera = await sourceView.getCamera();
await depthView.setCamera(camera); await depthView.setCamera(camera);
await depthView.setScene(projectionScene);
await depthView.setViewport(viewport.width, viewport.height);
await projectionView.setCamera(camera); await projectionView.setCamera(camera);
await (depthView as FFIView) await projectionView.setScene(projectionScene);
.setScene(await sourceView.getScene() as FFIScene);
await (projectionView as FFIView)
.setScene(await sourceView.getScene() as FFIScene);
await sourceView.setRenderOrder(0); var _pixelBuffers = <View, Uint8List>{};
await depthView.setRenderOrder(1);
await projectionView.setRenderOrder(2);
var pixelBuffers = await FilamentApp.instance!.capture(swapChain, if (renderSourceView) {
beforeRender: (view) async { _pixelBuffers[sourceView] =
if (view == depthView) { (await FilamentApp.instance!.capture(null, view: sourceView))
await target.setMaterialInstanceAt(depthWriteMaterialInstance); .first
} else if (view == projectionView) { .$2;
await target.setMaterialInstanceAt(projectionMaterialInstance);
}
},
pixelDataFormat: PixelDataFormat.RGBA,
pixelDataType: PixelDataType.FLOAT);
await target.setMaterialInstanceAt(originalMi);
_pixelBuffers.clear();
for (final (view, pixelBuffer) in pixelBuffers) {
_pixelBuffers[view] = pixelBuffer;
} }
await FilamentApp.instance!.unregister(swapChain, depthView); await target.setMaterialInstanceAt(depthWriteMaterialInstance);
await FilamentApp.instance!.unregister(swapChain, projectionView); _pixelBuffers[depthView] =
(await FilamentApp.instance!.capture(null, view: depthView)).first.$2;
await target.setMaterialInstanceAt(originalMi); await projectionMaterialInstance.setParameterTexture(
"color", texture as FFITexture, sampler);
await target.setMaterialInstanceAt(projectionMaterialInstance);
_pixelBuffers[projectionView] =
(await FilamentApp.instance!.capture(null, view: projectionView)).first.$2;
await target.setMaterialInstanceAt(originalMi as FFIMaterialInstance);
await sourceView.setScene(originalScene);
return TextureProjectionResult(_pixelBuffers[sourceView],
_pixelBuffers[depthView]!, _pixelBuffers[projectionView]!);
} }
} }
class TextureProjectionResult {
final Uint8List? sourceView;
final Uint8List depth;
final Uint8List projected;
TextureProjectionResult(this.sourceView, this.depth, this.projected);
}

13
thermion_dart/test/projection_tests.dart
View File

@@ -2,7 +2,6 @@
import 'dart:async'; import 'dart:async';
import 'dart:io'; import 'dart:io';
import 'dart:math'; import 'dart:math';
import 'dart:typed_data';
import 'package:test/test.dart'; import 'package:test/test.dart';
import 'package:thermion_dart/src/utils/src/texture_projection.dart'; import 'package:thermion_dart/src/utils/src/texture_projection.dart';
import 'package:thermion_dart/thermion_dart.dart'; import 'package:thermion_dart/thermion_dart.dart';
@@ -43,7 +42,7 @@ void main() async {
await camera.setLensProjection(near: 0.75, far: 100); await camera.setLensProjection(near: 0.75, far: 100);
final vp = await viewer.view.getViewport(); final vp = await viewer.view.getViewport();
final (width, height) = (vp.width, vp.height); final (width, height) = (vp.width, vp.height);
final dist = 5.0; final dist = 2.5;
await camera.lookAt( await camera.lookAt(
Vector3( Vector3(
-0.5, -0.5,
@@ -60,8 +59,7 @@ void main() async {
await ubershader.setParameterTexture( await ubershader.setParameterTexture(
"baseColorMap", originalTexture, sampler); "baseColorMap", originalTexture, sampler);
var textureProjection = var textureProjection = await TextureProjection.create(viewer.view);
await TextureProjection.create(viewer.view, testHelper.swapChain);
await FilamentApp.instance!.setClearOptions(0, 0, 0, 1, await FilamentApp.instance!.setClearOptions(0, 0, 0, 1,
clearStencil: 0, discard: false, clear: true); clearStencil: 0, discard: false, clear: true);
@@ -86,11 +84,12 @@ void main() async {
), ),
); );
await textureProjection.project(cube); final result = await textureProjection.project(
final depth = textureProjection.getDepthWritePixelBuffer(); await (await viewer.view.getRenderTarget())!.getColorTexture(), cube);
final depth = result.depth;
await savePixelBufferToBmp( await savePixelBufferToBmp(
depth, width, height, "${testHelper.outDir.path}/depth_$i.bmp"); depth, width, height, "${testHelper.outDir.path}/depth_$i.bmp");
final projected = textureProjection.getProjectedPixelBuffer(); final projected = result.projected;
await savePixelBufferToBmp(projected, width, height, await savePixelBufferToBmp(projected, width, height,
"${testHelper.outDir.path}/projected_$i.bmp"); "${testHelper.outDir.path}/projected_$i.bmp");
await cube.setMaterialInstanceAt(ubershader); await cube.setMaterialInstanceAt(ubershader);