From fae619feb89bd8aece6e56f1cf0aa9fd4e317b0d Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 26 Apr 2023 17:49:44 +0800 Subject: [PATCH 1/8] migrate some animation helper classes --- lib/animations/animation_builder.dart | 47 +++++++--- lib/animations/animations.dart | 61 ------------ lib/animations/bone_animation_data.dart | 16 ++++ lib/animations/bone_driver.dart | 57 ++++++++++++ lib/animations/csv_animation.dart | 113 +++++++++++++++++++++++ lib/animations/morph_animation_data.dart | 31 +++++++ test/bone_driver_test.dart | 73 +++++++++++++++ 7 files changed, 326 insertions(+), 72 deletions(-) delete mode 100644 lib/animations/animations.dart create mode 100644 lib/animations/bone_animation_data.dart create mode 100644 lib/animations/bone_driver.dart create mode 100644 lib/animations/csv_animation.dart create mode 100644 lib/animations/morph_animation_data.dart create mode 100644 test/bone_driver_test.dart diff --git a/lib/animations/animation_builder.dart b/lib/animations/animation_builder.dart index 4df2a40a..45455cac 100644 --- a/lib/animations/animation_builder.dart +++ b/lib/animations/animation_builder.dart @@ -1,13 +1,13 @@ -import 'package:polyvox_filament/animations/animations.dart'; +import 'package:polyvox_filament/animations/bone_animation_data.dart'; +import 'package:polyvox_filament/animations/morph_animation_data.dart'; import 'package:polyvox_filament/filament_controller.dart'; -import 'package:tuple/tuple.dart'; import 'package:flutter/foundation.dart'; import 'package:vector_math/vector_math.dart'; class AnimationBuilder { final FilamentController controller; - DartBoneAnimation? dartBoneAnimation; + BoneAnimationData? BoneAnimationData; double _frameLengthInMs = 0; double _duration = 0; @@ -16,7 +16,7 @@ class AnimationBuilder { double? _interpMorphStartValue; double? _interpMorphEndValue; - List? _dartBoneAnimations = null; + List? _BoneAnimationDatas = null; FilamentEntity asset; String meshName; @@ -53,11 +53,11 @@ class AnimationBuilder { } var morphAnimation = - MorphAnimation(meshName, morphData, morphNames, _frameLengthInMs); - print("SETTING!"); - controller.setMorphAnimation(asset, morphAnimation); - // return Tuple2>( - // morphAnimation, _dartBoneAnimations!); + MorphAnimationData(meshName, morphData, morphNames, _frameLengthInMs); + + controller.setMorphAnimationData(asset, morphAnimation); + // return Tuple2>( + // morphAnimation, _BoneAnimationDatas!); } AnimationBuilder setDuration(double secs) { @@ -118,15 +118,40 @@ class AnimationBuilder { // var boneFrameData = BoneTransformFrameData(translations, quats); - // _DartBoneAnimations ??= []; + // _BoneAnimationDatas ??= []; // var frameData = List>.generate( // numFrames, (index) => boneFrameData.getFrameData(index).toList()); // var animData = Float32List.fromList(frameData.expand((x) => x).toList()); - // _DartBoneAnimations!.add(DartDartBoneAnimation([boneName], [meshName], animData)); + // _BoneAnimationDatas!.add(DartBoneAnimationData([boneName], [meshName], animData)); return this; } } + +class BoneTransformFrameData { + final List translations; + final List quaternions; + + /// + /// The length of [translations] and [quaternions] must be the same; + /// each entry represents the Vec3/Quaternion for the given frame. + /// + BoneTransformFrameData(this.translations, this.quaternions) { + if (translations.length != quaternions.length) { + throw Exception("Length of translation/quaternion frames must match"); + } + } + + Iterable getFrameData(int frame) sync* { + yield translations[frame].x; + yield translations[frame].y; + yield translations[frame].z; + yield quaternions[frame].x; + yield quaternions[frame].y; + yield quaternions[frame].z; + yield quaternions[frame].w; + } +} diff --git a/lib/animations/animations.dart b/lib/animations/animations.dart deleted file mode 100644 index 41d1edd6..00000000 --- a/lib/animations/animations.dart +++ /dev/null @@ -1,61 +0,0 @@ -import 'dart:typed_data'; - -import 'package:vector_math/vector_math.dart'; - -class DartBoneAnimation { - final String boneName; - final String meshName; - final Float32List frameData; - double frameLengthInMs; - DartBoneAnimation( - this.boneName, this.meshName, this.frameData, this.frameLengthInMs); -} - -// -// Frame weights for the morph targets specified in [morphNames] attached to mesh [meshName]. -// morphData is laid out as numFrames x numMorphTargets -// where the weights are in the same order as [morphNames]. -// [morphNames] must be provided but is not used directly; this is only used to check that the eventual asset being animated contains the same morph targets in the same order. -// -class MorphAnimation { - final String meshName; - final List morphNames; - - final Float32List data; - - MorphAnimation( - this.meshName, this.data, this.morphNames, this.frameLengthInMs) { - assert(data.length == morphNames.length * numFrames); - } - - int get numMorphWeights => morphNames.length; - - int get numFrames => data.length ~/ numMorphWeights; - - final double frameLengthInMs; -} - -class BoneTransformFrameData { - final List translations; - final List quaternions; - - /// - /// The length of [translations] and [quaternions] must be the same; - /// each entry represents the Vec3/Quaternion for the given frame. - /// - BoneTransformFrameData(this.translations, this.quaternions) { - if (translations.length != quaternions.length) { - throw Exception("Length of translation/quaternion frames must match"); - } - } - - Iterable getFrameData(int frame) sync* { - yield translations[frame].x; - yield translations[frame].y; - yield translations[frame].z; - yield quaternions[frame].x; - yield quaternions[frame].y; - yield quaternions[frame].z; - yield quaternions[frame].w; - } -} diff --git a/lib/animations/bone_animation_data.dart b/lib/animations/bone_animation_data.dart new file mode 100644 index 00000000..cb694b01 --- /dev/null +++ b/lib/animations/bone_animation_data.dart @@ -0,0 +1,16 @@ +import 'dart:typed_data'; +import 'package:vector_math/vector_math.dart'; + +/// +/// Model class for bone animation frame data. +/// To create dynamic/runtime bone animations (as distinct from animations embedded in a glTF asset), create an instance containing the relevant +/// data and pass to the [setBoneAnimation] method on a [FilamentController]. +/// +class BoneAnimationData { + final String boneName; + final String meshName; + final Float32List frameData; + double frameLengthInMs; + BoneAnimationData( + this.boneName, this.meshName, this.frameData, this.frameLengthInMs); +} diff --git a/lib/animations/bone_driver.dart b/lib/animations/bone_driver.dart new file mode 100644 index 00000000..eeb0237a --- /dev/null +++ b/lib/animations/bone_driver.dart @@ -0,0 +1,57 @@ +import 'dart:convert'; +import 'package:vector_math/vector_math.dart'; +import 'package:flutter/foundation.dart'; +import 'package:vector_math/vector_math.dart'; + +/// +/// Some animation data may be specified as blendshape weights (say, between -1 and 1) +/// but at runtime we want to retarget this to drive a bone translation/rotation (say, between -pi/2 and pi/2). +/// A [BoneDriver] is our mechanism for translating the former to the latter, containing: +/// 1) a blendshape name +/// 2) a bone name +/// 3) min/max translation values (corresponding to -1/1 on the blendshape), and +/// 4) min/max rotation values (corresponding to -1/1 on the blendshape) +/// + +class BoneDriver { + final String bone; + final String blendshape; + + late final Vector3 transMin; + late final Vector3 transMax; + late final Quaternion rotMin; + late final Quaternion rotMax; + + BoneDriver(this.bone, this.blendshape, this.rotMin, this.rotMax, + Vector3? transMin, Vector3? transMax) { + this.transMin = transMin ?? Vector3.zero(); + this.transMax = transMax ?? Vector3.zero(); + } + + factory BoneDriver.fromJsonObject(dynamic jsonObject) { + return BoneDriver( + jsonObject["bone"], + jsonObject["blendshape"], + Quaternion.fromFloat32List(Float32List.fromList(jsonObject["rotMin"])), + Quaternion.fromFloat32List(Float32List.fromList(jsonObject["rotMax"])), + Vector3.fromFloat32List(Float32List.fromList(jsonObject["transMin"])), + Vector3.fromFloat32List(Float32List.fromList(jsonObject["transMax"])), + ); + } + + // + // Accepts a Float32List containing [numFrames] frames of data for a single morph target weight (for efficiency, this must be unravelled to a single contiguous Float32List). + // Returns a generator that yields [numFrames] Quaternions, each representing the (weighted) rotation/translation specified by the mapping of this BoneDriver. + // + Iterable transform(List morphTargetFrameData) sync* { + for (int i = 0; i < morphTargetFrameData.length; i++) { + var weight = (morphTargetFrameData[i] / 2) + 0.5; + + yield Quaternion( + rotMin.x + (weight * (rotMax.x - rotMin.x)), + rotMin.y + (weight * (rotMax.y - rotMin.y)), + rotMin.z + (weight * (rotMax.z - rotMin.z)), + 1.0); + } + } +} diff --git a/lib/animations/csv_animation.dart b/lib/animations/csv_animation.dart new file mode 100644 index 00000000..b46d35d5 --- /dev/null +++ b/lib/animations/csv_animation.dart @@ -0,0 +1,113 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; +import 'package:tuple/tuple.dart'; +import 'package:polyvox_filament/animations/bone_animation_data.dart'; +import 'package:polyvox_filament/animations/bone_driver.dart'; +import 'package:polyvox_filament/animations/morph_animation_data.dart'; +import 'package:vector_math/vector_math.dart'; + +/// +/// A class for loading animation data from a single CSV and allocating between morph/bone animation with help. +/// +class DynamicAnimation { + final MorphAnimationData morphAnimation; + final List boneAnimation; + + factory DynamicAnimation.load(String meshName, String csvPath, + {String? boneDriverConfigPath}) { + // create a MorphAnimationData instance from the given CSV + var llf = _loadLiveLinkFaceCSV(csvPath); + var morphNames = llf + .item1; //.where((name) => !boneDrivers.any((element) => element.blendshape == name)); + var morphAnimationData = MorphAnimationData( + meshName, + llf.item2, + morphNames, + 1000 / 60.0, + ); + + final boneAnimations = []; + + // if applicable, load the bone driver config + if (boneDriverConfigPath != null) { + var boneData = json.decode(File(boneDriverConfigPath).readAsStringSync()); + // for each driver + for (var key in boneData.keys()) { + var driver = BoneDriver.fromJsonObject(boneData[key]); + + // get all frames for the single the blendshape + var morphFrameData = + morphAnimationData.getData(driver.blendshape).toList(); + + // apply the driver to the blendshape weight + var transformedQ = driver.transform(morphFrameData).toList(); + + // transform the quaternion to a Float32List + var transformedF = _quaternionToFloatList(transformedQ); + + // add to the list of boneAnimations + boneAnimations.add(BoneAnimationData( + driver.bone, meshName, transformedF, 1000.0 / 60.0)); + } + } + + return DynamicAnimation(morphAnimationData, boneAnimations); + } + + static Float32List _quaternionToFloatList(List quats) { + var data = Float32List(quats.length * 4); + for (var quat in quats) { + data.addAll([0, 0, 0, quat.w, quat.x, quat.y, quat.z]); + } + return data; + } + + DynamicAnimation(this.morphAnimation, this.boneAnimation); + + /// + /// Load visemes fom a CSV file formatted according to the following header: + /// "Timecode,BlendShapeCount,EyeBlinkLeft,EyeLookDownLeft,EyeLookInLeft,EyeLookOutLeft,EyeLookUpLeft,EyeSquintLeft,EyeWideLeft,EyeBlinkRight,EyeLookDownRight,EyeLookInRight,EyeLookOutRight,EyeLookUpRight,EyeSquintRight,EyeWideRight,JawForward,JawRight,JawLeft,JawOpen,MouthClose,MouthFunnel,MouthPucker,MouthRight,MouthLeft,MouthSmileLeft,MouthSmileRight,MouthFrownLeft,MouthFrownRight,MouthDimpleLeft,MouthDimpleRight,MouthStretchLeft,MouthStretchRight,MouthRollLower,MouthRollUpper,MouthShrugLower,MouthShrugUpper,MouthPressLeft,MouthPressRight,MouthLowerDownLeft,MouthLowerDownRight,MouthUpperUpLeft,MouthUpperUpRight,BrowDownLeft,BrowDownRight,BrowInnerUp,BrowOuterUpLeft,BrowOuterUpRight,CheekPuff,CheekSquintLeft,CheekSquintRight,NoseSneerLeft,NoseSneerRight,TongueOut,HeadYaw,HeadPitch,HeadRoll,LeftEyeYaw,LeftEyePitch,LeftEyeRoll,RightEyeYaw,RightEyePitch,RightEyeRoll" + /// Returns only those specified by [targetNames]. + /// + static Tuple2, Float32List> _loadLiveLinkFaceCSV(String path) { + final data = File(path) + .readAsLinesSync() + .where((l) => l.length > 1) + .map((l) => l.split(",")); + + final header = data.first; + final numBlendShapes = header.length - 2; + + final _data = []; + + for (var frame in data.skip(1)) { + int numFrameWeights = frame.length - 2; + // CSVs may contain rows where the "BlendShapeCount" column is set to "0" and/or the weight columns are simply missing. + // This can happen when something went wrong while recording via an app (e.g. LiveLinkFace) + // Whenever we encounter this type of row, we consider that all weights should be set to zero for that frame. + if (numFrameWeights == int.parse(frame[1])) { + _data.addAll(List.filled(numBlendShapes, 0.0)); + continue; + } + + // + // Now, we check that the actual number of weight columns matches the header + // we ignore the "BlendShapeCount" column (and just count the number of columns) + // This is due to some legacy issues where we generated CSVs that had 61 weight columns, but accidentally left the "BlendShapeCount" column at 55 + // This is probably fine as we always have either zero weights (handled above), or all weights (handled below). + // In other words, if this throws, we have a serious problem. + if (numFrameWeights != numBlendShapes) { + throw Exception( + "Malformed CSV, header specifies ${numBlendShapes} columns but frame specified only $numFrameWeights weights"); + } + + _data.addAll(frame + .skip(2) + .map((weight) => double.parse(weight)) + .cast() + .toList()); + } + return Tuple2(header.skip(2).toList(), Float32List.fromList(_data)); + } +} diff --git a/lib/animations/morph_animation_data.dart b/lib/animations/morph_animation_data.dart new file mode 100644 index 00000000..ac6ab4a2 --- /dev/null +++ b/lib/animations/morph_animation_data.dart @@ -0,0 +1,31 @@ +// +// Frame weights for the morph targets specified in [morphNames] attached to mesh [meshName]. +// morphData is laid out as numFrames x numMorphTargets +// where the weights are in the same order as [morphNames]. +// [morphNames] must be provided but is not used directly; this is only used to check that the eventual asset being animated contains the same morph targets in the same order. +// +import 'dart:typed_data'; + +class MorphAnimationData { + final String meshName; + final List morphNames; + + final Float32List data; + + MorphAnimationData( + this.meshName, this.data, this.morphNames, this.frameLengthInMs) { + assert(data.length == morphNames.length * numFrames); + } + + int get numMorphWeights => morphNames.length; + + int get numFrames => data.length ~/ numMorphWeights; + + final double frameLengthInMs; + + Iterable getData(String morphName) sync* { + for (int i = 0; i < numFrames; i++) { + yield data[i * numMorphWeights]; + } + } +} diff --git a/test/bone_driver_test.dart b/test/bone_driver_test.dart new file mode 100644 index 00000000..09a90305 --- /dev/null +++ b/test/bone_driver_test.dart @@ -0,0 +1,73 @@ +import 'dart:convert'; +import 'dart:math'; +import 'dart:typed_data'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:polyvox_filament/animations/bone_driver.dart'; +import 'package:vector_math/vector_math.dart'; + +void main() { + group('BoneDriver', () { + test('constructor sets correct values', () { + Quaternion rotMin = Quaternion.identity(); + Quaternion rotMax = Quaternion.axisAngle(Vector3(1, 0, 0), 0.5); + Vector3 transMin = Vector3.zero(); + Vector3 transMax = Vector3(1, 1, 1); + + BoneDriver boneDriver = BoneDriver( + 'bone1', 'blendshape1', rotMin, rotMax, transMin, transMax); + + expect(boneDriver.bone, 'bone1'); + expect(boneDriver.blendshape, 'blendshape1'); + expect(boneDriver.rotMin, rotMin); + expect(boneDriver.rotMax, rotMax); + expect(boneDriver.transMin, transMin); + expect(boneDriver.transMax, transMax); + }); + + test('fromJsonObject creates BoneDriver instance correctly', () { + dynamic jsonObject = { + "bone": "bone1", + "blendshape": "blendshape1", + "rotMin": Quaternion.identity().storage, + "rotMax": Quaternion.axisAngle(Vector3(1, 0, 0), 0.5).storage, + "transMin": Vector3.zero().storage, + "transMax": Vector3(1, 1, 1).storage + }; + + BoneDriver boneDriver = BoneDriver.fromJsonObject(jsonObject); + + expect(boneDriver.bone, 'bone1'); + expect(boneDriver.blendshape, 'blendshape1'); + expect(boneDriver.rotMin.absoluteError(Quaternion.identity()), 0); + expect( + boneDriver.rotMax + .absoluteError(Quaternion.axisAngle(Vector3(1, 0, 0), 0.5)), + 0); + expect(boneDriver.transMin.absoluteError(Vector3.zero()), 0); + expect(boneDriver.transMax.absoluteError(Vector3(1, 1, 1)), 0); + }); + + test('transform generates correct Quaternions', () { + Quaternion rotMin = Quaternion.identity(); + Quaternion rotMax = Quaternion.axisAngle(Vector3(1, 0, 0), 0.5); + BoneDriver boneDriver = + BoneDriver('bone1', 'blendshape1', rotMin, rotMax, null, null); + + List morphTargetFrameData = [-1, 0, 1]; + List expectedResult = [ + Quaternion(rotMin.x, rotMin.y, rotMin.z, 1.0), + Quaternion((rotMin.x + rotMax.x) / 2, (rotMin.y + rotMax.y) / 2, + (rotMin.z + rotMax.z) / 2, 1.0), + Quaternion(rotMax.x, rotMax.y, rotMax.z, 1.0), + ]; + + Iterable result = boneDriver.transform(morphTargetFrameData); + List resultAsList = result.toList(); + expect(resultAsList.length, expectedResult.length); + + for (int i = 0; i < expectedResult.length; i++) { + expect(resultAsList[i].absoluteError(expectedResult[i]), 0); + } + }); + }); +} From 19f7e778ba664166ef08988ee984d416c69d7c74 Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 26 Apr 2023 17:50:01 +0800 Subject: [PATCH 2/8] remove old transform manager --- lib/transform_manager.dart | 63 -------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 lib/transform_manager.dart diff --git a/lib/transform_manager.dart b/lib/transform_manager.dart deleted file mode 100644 index d9a21ee1..00000000 --- a/lib/transform_manager.dart +++ /dev/null @@ -1,63 +0,0 @@ - -// import 'package:flutter/widgets.dart'; -// import 'package:polyvox_filament/filament_controller.dart'; -// import 'package:polyvox_filament/filament_controller.dart'; -// import 'package:vector_math/vector_math_64.dart'; - -// class Position { -// final double x; -// final double y; -// final double z; -// Position(this.x, this.y, this.z); -// Position copy({double? x, double? y, double? z}) { -// return Position(x ?? this.x, y ?? this.y, z ?? this.z); -// } - -// factory Position.zero() { -// return Position(0,0,0); -// } -// } - -// class Rotation { -// final double rads; -// final double x; -// final double y; -// final double z; -// Rotation(this.x, this.y, this.z, this.rads); -// Rotation copy({double? rads, double? x, double? y, double? z}) { -// return Rotation(x ?? this.x, y ?? this.y, z ?? this.z, rads ?? this.rads); -// } - -// factory Rotation.zero() { -// return Rotation(0, 1,0,0); -// } - -// } - - - -// /// -// /// Handles local transforms for assets/cameras. -// /// -// class TransformManager { - -// final FilamentController _controller; - -// Matrix4 transform = Matrix4.identity(); - -// TransformManager(this._controller); - -// void scale(double scale) { -// transform.scale(scale, scale, scale); -// } - - -// void translate(double x, double y, double z) { -// transform.translate(x,y,z); -// } - -// void rotate(double x, double y, double z) { -// transform.rotate(Vector3(x,y,z)); -// } - -// } \ No newline at end of file From c0a7cea1d15206fd2bb506de55f66e7fb4ea8885 Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 26 Apr 2023 17:50:06 +0800 Subject: [PATCH 3/8] remove old test --- test/mimetic_filament_test.dart | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 test/mimetic_filament_test.dart diff --git a/test/mimetic_filament_test.dart b/test/mimetic_filament_test.dart deleted file mode 100644 index 20fc1dcc..00000000 --- a/test/mimetic_filament_test.dart +++ /dev/null @@ -1,10 +0,0 @@ -import 'package:flutter/services.dart'; -import 'package:flutter_test/flutter_test.dart'; - -void main() { - const MethodChannel channel = MethodChannel('polyvox_filament'); - - TestWidgetsFlutterBinding.ensureInitialized(); - - test('getPlatformVersion', () async {}); -} From 6641403187afbd3695c42aec167fc7eb8416bede Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 26 Apr 2023 17:50:26 +0800 Subject: [PATCH 4/8] animation data class renaming --- lib/filament_controller.dart | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/filament_controller.dart b/lib/filament_controller.dart index 1fa65cb7..1e6c6dad 100644 --- a/lib/filament_controller.dart +++ b/lib/filament_controller.dart @@ -9,10 +9,10 @@ import 'package:flutter/animation.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter/services.dart'; +import 'package:polyvox_filament/animations/bone_animation_data.dart'; +import 'package:polyvox_filament/animations/morph_animation_data.dart'; import 'package:polyvox_filament/generated_bindings.dart'; -import 'animations/animations.dart'; - typedef AssetManager = Pointer; typedef FilamentViewer = Pointer; typedef FilamentEntity = int; @@ -299,7 +299,8 @@ class FilamentController { /// [morphWeights] is a list of doubles in frame-major format. /// Each frame is [numWeights] in length, and each entry is the weight to be applied to the morph target located at that index in the mesh primitive at that frame. /// - void setMorphAnimation(FilamentEntity asset, MorphAnimation animation) async { + void setMorphAnimationData( + FilamentEntity asset, MorphAnimationData animation) async { var data = calloc(animation.data.length); for (int i = 0; i < animation.data.length; i++) { data.elementAt(i).value = animation.data[i]; @@ -321,7 +322,7 @@ class FilamentController { /// Each frame is [numWeights] in length, and each entry is the weight to be applied to the morph target located at that index in the mesh primitive at that frame. /// void setBoneAnimation( - FilamentEntity asset, List animations) async { + FilamentEntity asset, List animations) async { var data = calloc(animations.length * animations.first.frameData.length); int offset = 0; From a10825c2ac5040f96f55c33bb9e56a5e590ea0f6 Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 26 Apr 2023 17:50:44 +0800 Subject: [PATCH 5/8] comment TODO re OpenGL tex coords --- lib/filament_widget.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/filament_widget.dart b/lib/filament_widget.dart index b601d8b4..ea8b42ef 100644 --- a/lib/filament_widget.dart +++ b/lib/filament_widget.dart @@ -118,7 +118,8 @@ class _FilamentWidgetState extends State { ? Container() : Transform( alignment: Alignment.center, - transform: Matrix4.rotationX(pi), + transform: Matrix4.rotationX( + pi), // TODO - this rotation is due to OpenGL texture coordinate working in a different space from Flutter, can we move this to the C++ side somewhere? child: texture) : texture)))); }); From d1e15b53c5880bebeb4fd83f6d69947b1a28a7b4 Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 26 Apr 2023 17:51:50 +0800 Subject: [PATCH 6/8] remove RB assignment operator and carve out C++ only sections --- ios/include/ResourceBuffer.hpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ios/include/ResourceBuffer.hpp b/ios/include/ResourceBuffer.hpp index b904acce..046caf5d 100644 --- a/ios/include/ResourceBuffer.hpp +++ b/ios/include/ResourceBuffer.hpp @@ -27,17 +27,11 @@ extern "C" { // struct ResourceBuffer { #if defined(__cplusplus) - ResourceBuffer(const void* data, const uint32_t size, const uint32_t id) : data(data), size(size), id(id) {}; - ResourceBuffer& operator=(ResourceBuffer other) { - data = other.data; - size = other.size; - id = other.id; - return *this; - } + ResourceBuffer(const void* const data, const uint32_t size, const uint32_t id) : data(data), size(size), id(id) {}; #endif - const void* data; - uint32_t size; - uint32_t id; + const void * const data; + const uint32_t size; + const uint32_t id; }; typedef struct ResourceBuffer ResourceBuffer; From 62c4be05630a39786f0c3b21dff7bed33c06ecc3 Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Thu, 27 Apr 2023 16:32:32 +0800 Subject: [PATCH 7/8] fix dynamic bone animations --- example/lib/main.dart | 34 +- ios/include/AssetManager.hpp | 34 +- ios/include/PolyvoxFilamentApi.h | 21 +- ios/include/SceneAsset.hpp | 56 +-- ios/include/material/FileMaterialProvider.hpp | 2 +- ios/src/AssetManager.cpp | 327 ++++++++++-------- ios/src/PolyvoxFilamentApi.cpp | 15 +- lib/animations/animation_builder.dart | 4 +- lib/animations/bone_animation_data.dart | 1 + lib/animations/bone_driver.dart | 79 +++-- lib/animations/csv_animation.dart | 50 ++- .../live_link_face_bone_driver.dart | 12 + lib/animations/morph_animation_data.dart | 3 +- lib/filament_controller.dart | 20 +- lib/generated_bindings.dart | 291 +++++++--------- test/bone_driver_test.dart | 109 +++--- 16 files changed, 538 insertions(+), 520 deletions(-) create mode 100644 lib/animations/live_link_face_bone_driver.dart diff --git a/example/lib/main.dart b/example/lib/main.dart index 3c6d9654..db424975 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,12 +1,13 @@ import 'dart:math'; +import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:vector_math/vector_math.dart' as v; import 'package:polyvox_filament/filament_controller.dart'; +import 'package:polyvox_filament/animations/bone_animation_data.dart'; import 'package:polyvox_filament/filament_gesture_detector.dart'; import 'package:polyvox_filament/filament_widget.dart'; import 'package:polyvox_filament/animations/animation_builder.dart'; -import 'package:polyvox_filament/animations/animations.dart'; void main() { runApp(const MyApp()); @@ -101,6 +102,7 @@ class _MyAppState extends State with SingleTickerProviderStateMixin { break; case 9: for (int i = 0; i < _animationNames.length; i++) { + print("Playing animation ${_animationNames[i]}"); _filamentController.playAnimation(_cube!, i, loop: _loop); } @@ -202,15 +204,25 @@ class _MyAppState extends State with SingleTickerProviderStateMixin { _filamentController.clearLights(); break; case 32: - _filamentController.setCameraModelMatrix(List.filled(16, 1.0)); + var frameData = Float32List.fromList( + List.generate(120, (i) => i / 120).expand((x) { + var vals = List.filled(7, x); + vals[3] = 1.0; + // vals[4] = 0; + vals[5] = 0; + vals[6] = 0; + return vals; + }).toList()); - // _filamentController.setBoneTransform( - // _cube!, - // "Bone.001", - // "Cube.001", - // BoneTransform([Vec3(x: 0, y: 0.0, z: 0.0)], - // [Quaternion(x: 1, y: 1, z: 1, w: 1)])); - break; + _filamentController.setBoneAnimation(_cube!, [ + BoneAnimationData("Bone.001", "Cube.001", frameData, 1000.0 / 60.0) + ]); + // , + // "Bone.001", + // "Cube.001", + // BoneTransform([Vec3(x: 0, y: 0.0, z: 0.0)], + // [Quaternion(x: 1, y: 1, z: 1, w: 1)])); + // break; } } @@ -260,9 +272,7 @@ class _MyAppState extends State with SingleTickerProviderStateMixin { _item(value: 21, child: Text('swap cube texture')), _item(value: 22, child: Text('transform to unit cube')), _item(value: 23, child: Text('set position to 1, 1, -1')), - _item( - value: 32, - child: Text('set bone transform to 1, 1, -1')), + _item(value: 32, child: Text('construct bone animation')), _item(value: 24, child: Text('rotate by pi around Y axis')), _item(value: 5, child: Text('load flight helmet')), _item(value: 6, child: Text('remove cube')), diff --git a/ios/include/AssetManager.hpp b/ios/include/AssetManager.hpp index 84b8a35b..7b2da476 100644 --- a/ios/include/AssetManager.hpp +++ b/ios/include/AssetManager.hpp @@ -40,21 +40,22 @@ namespace polyvox { size_t getLightEntityCount(EntityId e) const noexcept; void updateAnimations(); - bool setBoneAnimationBuffer( - EntityId entity, - int length, - const char** const boneNames, - const char** const meshNames, - const float* const frameData, - int numFrames, - float frameLengthInMs); + bool setMorphAnimationBuffer( - EntityId entity, - const char* entityName, - const float* const morphData, + EntityId entityId, + const char* entityName, + const float* const morphData, int numMorphWeights, int numFrames, float frameLengthInMs); + bool setBoneAnimationBuffer( + EntityId entity, + const float* const frameData, + int numFrames, + int numBones, + const char** const boneNames, + const char* const meshName, + float frameLengthInMs); void playAnimation(EntityId e, int index, bool loop, bool reverse); void stopAnimation(EntityId e, int index); void setMorphTargetWeights(const char* const entityName, float *weights, int count); @@ -74,18 +75,15 @@ namespace polyvox { vector _assets; tsl::robin_map _entityIdLookup; - void setBoneTransform( - FilamentInstance* instance, - vector animations, - int frameNumber - ); - utils::Entity findEntityByName( SceneAsset asset, const char* entityName ); - inline void updateTransform(SceneAsset asset); + inline void updateTransform(SceneAsset& asset); + + inline void setBoneTransform(SceneAsset& asset, int frameNumber); + }; diff --git a/ios/include/PolyvoxFilamentApi.h b/ios/include/PolyvoxFilamentApi.h index 01eae3e6..8f92756f 100644 --- a/ios/include/PolyvoxFilamentApi.h +++ b/ios/include/PolyvoxFilamentApi.h @@ -57,29 +57,16 @@ bool set_morph_animation( int numFrames, float frameLengthInMs); - void set_bone_animation( +void set_bone_animation( void* assetManager, EntityId asset, - int length, - const char** const boneNames, - const char** const meshNames, const float* const frameData, int numFrames, + int numBones, + const char** const boneNames, + const char* const meshName, float frameLengthInMs); -// void set_bone_transform( -// EntityId asset, -// const char* boneName, -// const char* entityName, -// float transX, -// float transY, -// float transZ, -// float quatX, -// float quatY, -// float quatZ, -// float quatW -// ); - void play_animation(void* assetManager, EntityId asset, int index, bool loop, bool reverse); void set_animation_frame(void* assetManager, EntityId asset, int animationIndex, int animationFrame); void stop_animation(void* assetManager, EntityId asset, int index); diff --git a/ios/include/SceneAsset.hpp b/ios/include/SceneAsset.hpp index 616cbd9f..1eb93924 100644 --- a/ios/include/SceneAsset.hpp +++ b/ios/include/SceneAsset.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -36,35 +37,6 @@ namespace polyvox { bool mReverse = false; float mDuration = 0; bool mAnimating = false; - - // AnimationStatus() { - // Log("default constr"); - // } - - // AnimationStatus(AnimationStatus& a) { - // mStart = a.mStart; - // mLoop = a.mLoop; - // mReverse = a.mReverse; - // mDuration = a.mDuration; - // mFrameNumber = a.mFrameNumber; - // } - - // AnimationStatus& operator=(AnimationStatus a) { - // mStart = a.mStart; - // mLoop = a.mLoop; - // mReverse = a.mReverse; - // mDuration = a.mDuration; - // mFrameNumber = a.mFrameNumber; - // return *this; - // } - - // AnimationStatus(AnimationStatus&& a) { - // mStart = a.mStart; - // mLoop = a.mLoop; - // mReverse = a.mReverse; - // mDuration = a.mDuration; - // mFrameNumber = a.mFrameNumber; - // } }; // @@ -78,24 +50,18 @@ namespace polyvox { int mNumMorphWeights = 0; }; - /// - /// Frame data for the bones/meshes specified by [mBoneIndices] and [mMeshTargets]. - /// This is mainly used as a wrapper for animation data being transferred from the Dart to the native side. - /// - struct BoneAnimationData { - size_t skinIndex = 0; - uint8_t mBoneIndex; - utils::Entity mMeshTarget; - vector mFrameData; - }; - // - // Use this to manually construct a buffer of frame data for bone animations. + // Use this to construct a dynamic (i.e. non-glTF embedded) bone animation. + // Only a single animation is supported at any time (i.e you can't blend animations). + // Multiple bones are supported but these must be skinned to a single mesh target. // struct BoneAnimationBuffer { + utils::Entity mMeshTarget; + vector mBones; + size_t skinIndex = 0; int mNumFrames = -1; float mFrameLengthInMs = 0; - vector mAnimations; + vector mFrameData; }; struct SceneAsset { @@ -129,10 +95,12 @@ namespace polyvox { mAnimations.resize(2 + mAnimator->getAnimationCount()); - for(int i=2; i < mAnimations.size(); i++) { - mAnimations[i].mDuration = mAnimator->getAnimationDuration(i-2); + for(int i=0; i < mAnimations.size() - 2; i++) { + mAnimations[i].mDuration = mAnimator->getAnimationDuration(i); } } + + }; } diff --git a/ios/include/material/FileMaterialProvider.hpp b/ios/include/material/FileMaterialProvider.hpp index f4a41c2e..606028a1 100644 --- a/ios/include/material/FileMaterialProvider.hpp +++ b/ios/include/material/FileMaterialProvider.hpp @@ -18,7 +18,7 @@ namespace polyvox { public: - FileMaterialProvider(Engine* engine, void* const data, size_t size) { + FileMaterialProvider(Engine* engine, const void* const data, const size_t size) { _m = Material::Builder() .package(data, size) .build(*engine); diff --git a/ios/src/AssetManager.cpp b/ios/src/AssetManager.cpp index 80995664..5c1f83cb 100644 --- a/ios/src/AssetManager.cpp +++ b/ios/src/AssetManager.cpp @@ -52,8 +52,13 @@ AssetManager::AssetManager(ResourceLoaderWrapper* resourceLoaderWrapper, _ubershaderProvider = gltfio::createUbershaderProvider( _engine, UBERARCHIVE_DEFAULT_DATA, UBERARCHIVE_DEFAULT_SIZE); EntityManager &em = EntityManager::get(); - _assetLoader = AssetLoader::create({_engine, _ubershaderProvider, _ncm, &em }); + _unlitProvider = new UnlitMaterialProvider(_engine); + + // auto rb = _resourceLoaderWrapper->load("file:///mnt/hdd_2tb/home/hydroxide/projects/polyvox/flutter/polyvox_filament/materials/toon.filamat"); + // auto toonProvider = new FileMaterialProvider(_engine, rb.data, (size_t) rb.size); + + _assetLoader = AssetLoader::create({_engine, _ubershaderProvider, _ncm, &em }); _gltfResourceLoader->addTextureProvider("image/png", _stbDecoder); _gltfResourceLoader->addTextureProvider("image/jpeg", _stbDecoder); @@ -213,13 +218,43 @@ void AssetManager::updateAnimations() { RenderableManager &rm = _engine->getRenderableManager(); for (auto& asset : _assets) { + if(!asset.mAnimating) { continue; } asset.mAnimating = false; + // GLTF animations + for(int j = 0; j < asset.mAnimations.size() - 2; j++) { + AnimationStatus& anim = asset.mAnimations[j]; + + if(!anim.mAnimating) { + // Log("Skipping anim at %d", j); + continue; + } + + auto elapsed = float(std::chrono::duration_cast(now - anim.mStart).count()) / 1000.0f; + + if(anim.mLoop || elapsed < anim.mDuration) { + asset.mAnimator->applyAnimation(j, elapsed); + asset.mAnimating = true; + // Log("Applying at %f", elapsed); + } else if(elapsed - anim.mDuration < 0.3) { + // cross-fade + // animator->applyCrossFade(j-2, anim.mDuration - 0.05, elapsed / 0.3); + // asset.mAnimating = true; + // anim.mStart = time_point_t::max(); + } else { + // stop + anim.mStart = time_point_t::max(); + anim.mAnimating = false; + Log("Finished"); + } + asset.mAnimator->updateBoneMatrices(); + } + // dynamically constructed morph animation - AnimationStatus& morphAnimation = asset.mAnimations[0]; + AnimationStatus& morphAnimation = asset.mAnimations[asset.mAnimations.size() - 2]; if(morphAnimation.mAnimating) { @@ -254,70 +289,109 @@ void AssetManager::updateAnimations() { } } - // // bone animation - // AnimationStatus boneAnimation = asset.mAnimations[1]; - // elapsed = (now - boneAnimation.mStart).count(); + // bone animation + AnimationStatus boneAnimation = asset.mAnimations[asset.mAnimations.size() - 1]; + if(boneAnimation.mAnimating) { + auto elapsed = float( + std::chrono::duration_cast( + now - boneAnimation.mStart + ).count()) / 1000.0f; + int lengthInFrames = static_cast( + boneAnimation.mDuration * 1000.0f / + asset.mBoneAnimationBuffer.mFrameLengthInMs + ); - // lengthInFrames = static_cast(boneAnimation.mDuration / asset.mBoneAnimationBuffer.mFrameLengthInMs); + // if more time has elapsed than the animation duration && not looping + // mark the animation as complete + if(elapsed >= boneAnimation.mDuration && !boneAnimation.mLoop) { + boneAnimation.mStart = time_point_t::max(); + boneAnimation.mAnimating = false; + Log("FINISHED"); + } else { - // if(elapsed >= boneAnimation.mDuration) { - // if(boneAnimation.mLoop) { - // boneAnimation.mStart = now; - // if(boneAnimation.mReverse) { - // boneAnimation.mFrameNumber = lengthInFrames; - // } - // asset.mAnimating = true; - // } else { - // boneAnimation.mStart = time_point_t::max(); - // } - // } else { - // asset.mAnimating = true; - // } + asset.mAnimating = true; + int frameNumber = static_cast(elapsed * 1000.0f / asset.mBoneAnimationBuffer.mFrameLengthInMs) % lengthInFrames; - // frameNumber = static_cast(elapsed / asset.mBoneAnimationBuffer.mFrameLengthInMs); - // if(frameNumber < lengthInFrames) { - // if(boneAnimation.mReverse) { - // frameNumber = lengthInFrames - frameNumber; - // } - // boneAnimation.mFrameNumber = frameNumber; - // setBoneTransform( - // asset.mAsset->getInstance(), - // asset.mBoneAnimationBuffer.mAnimations, - // frameNumber - // ); - // } + // offset from the end if reverse + if(boneAnimation.mReverse) { + frameNumber = lengthInFrames - frameNumber; + } - // GLTF animations - - int j = -1; - for(AnimationStatus& anim : asset.mAnimations) { - j++; - if(j < 2) { - continue; - } - - if(!anim.mAnimating) { - continue; - } - - auto elapsed = float(std::chrono::duration_cast(now - anim.mStart).count()) / 1000.0f; - - if(anim.mLoop || elapsed < anim.mDuration) { - asset.mAnimator->applyAnimation(j-2, elapsed); - asset.mAnimating = true; - } else if(elapsed - anim.mDuration < 0.3) { - // cross-fade - // animator->applyCrossFade(j-2, anim.mDuration - 0.05, elapsed / 0.3); - // asset.mAnimating = true; - // anim.mStart = time_point_t::max(); - } else { - // stop - anim.mStart = time_point_t::max(); + setBoneTransform( + asset, + frameNumber + ); } } - if(asset.mAnimating) { - asset.mAnimator->updateBoneMatrices(); - } + } +} + + void AssetManager::setBoneTransform(SceneAsset& asset, int frameNumber) { + + RenderableManager& rm = _engine->getRenderableManager(); + + const auto& filamentInstance = asset.mAsset->getInstance(); + + TransformManager &transformManager = _engine->getTransformManager(); + + int skinIndex = 0; + + math::mat4f inverseGlobalTransform = inverse( + transformManager.getWorldTransform( + transformManager.getInstance(asset.mBoneAnimationBuffer.mMeshTarget) + ) + ); + + auto renderable = rm.getInstance(asset.mBoneAnimationBuffer.mMeshTarget); + + for(int i = 0; i < asset.mBoneAnimationBuffer.mBones.size(); i++) { + auto mBoneIndex = asset.mBoneAnimationBuffer.mBones[i]; + auto frameDataOffset = (frameNumber * asset.mBoneAnimationBuffer.mBones.size() * 7) + asset.mBoneAnimationBuffer.mBones[i]; + + utils::Entity joint = filamentInstance->getJointsAt(skinIndex)[mBoneIndex]; + if(joint.isNull()) { + Log("ERROR : joint not found"); + continue; + } + // RenderableManager::Bone bone { math::quatf{ + // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6], + // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3], + // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4], + // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5] + // }, + // math::float3 { + // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+0], + // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+1], + // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+2] + // } + + // }; + // rm.setBones( + // renderable, + // &bone, + // 1, + // mBoneIndex + // ); + const math::mat4f localTransform(math::quatf{ + asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3], + asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4], + asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5], + asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6] + }); + const math::mat4f& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(skinIndex)[mBoneIndex]; + auto jointInstance = transformManager.getInstance(joint); + math::mat4f globalJointTransform = transformManager.getWorldTransform(jointInstance); + + math::mat4f boneTransform = inverseGlobalTransform * globalJointTransform * inverseBindMatrix * localTransform; + + rm.setBones( + renderable, + &boneTransform, + 1, + mBoneIndex + ); + + } } @@ -394,7 +468,7 @@ bool AssetManager::setMorphAnimationBuffer( asset.mMorphAnimationBuffer.mFrameLengthInMs = frameLengthInMs; asset.mMorphAnimationBuffer.mNumMorphWeights = numMorphWeights; - AnimationStatus& animation = asset.mAnimations[0]; + AnimationStatus& animation = asset.mAnimations[asset.mAnimations.size() - 2]; animation.mDuration = (frameLengthInMs * numFrames) / 1000.0f; animation.mStart = high_resolution_clock::now(); animation.mAnimating = true; @@ -407,21 +481,27 @@ bool AssetManager::setMorphAnimationBuffer( } bool AssetManager::setBoneAnimationBuffer( - EntityId entity, - int length, - const char** const boneNames, - const char** const meshNames, + EntityId entityId, const float* const frameData, int numFrames, + int numBones, + const char** const boneNames, + const char* const meshName, float frameLengthInMs) { - const auto& pos = _entityIdLookup.find(entity); + const auto& pos = _entityIdLookup.find(entityId); if(pos == _entityIdLookup.end()) { Log("ERROR: asset not found for entity."); return false; } auto& asset = _assets[pos->second]; + auto entity = findEntityByName(asset, meshName); + if(!entity) { + Log("Mesh target %s for bone animation could not be found", meshName); + return false; + } + auto filamentInstance = asset.mAsset->getInstance(); size_t skinCount = filamentInstance->getSkinCount(); @@ -433,92 +513,60 @@ bool AssetManager::setBoneAnimationBuffer( int skinIndex = 0; const utils::Entity* joints = filamentInstance->getJointsAt(skinIndex); size_t numJoints = filamentInstance->getJointCountAt(skinIndex); - vector boneIndices; - for(int i = 0; i < length; i++) { + + asset.mBoneAnimationBuffer.mBones.clear(); + for(int i = 0; i < numBones; i++) { + Log("Bone %s", boneNames[i]); for(int j = 0; j < numJoints; j++) { const char* jointName = _ncm->getName(_ncm->getInstance(joints[j])); - if(strcmp(jointName, boneNames[i]) == 0) { - boneIndices.push_back(j); + if(strcmp(jointName, boneNames[i]) == 0) { + asset.mBoneAnimationBuffer.mBones.push_back(j); break; } } } - if(boneIndices.size() != length) { + if(asset.mBoneAnimationBuffer.mBones.size() != numBones) { Log("Failed to find one or more bone indices"); return false; } - - asset.mBoneAnimationBuffer.mAnimations.clear(); - for(int i = 0; i < length; i++) { - BoneAnimationData boneAnimationData; - boneAnimationData.mBoneIndex = boneIndices[i]; + asset.mBoneAnimationBuffer.mFrameData.clear(); + // 7 == locX, locY, locZ, rotW, rotX, rotY, rotZ + asset.mBoneAnimationBuffer.mFrameData.resize(numFrames * numBones * 7); + asset.mBoneAnimationBuffer.mFrameData.insert( + asset.mBoneAnimationBuffer.mFrameData.begin(), + frameData, + frameData + numFrames * numBones * 7 + ); - auto entity = findEntityByName(asset, meshNames[i]); + Log("%d frames for %d bones", numFrames, numBones); - if(!entity) { - Log("Mesh target %s for bone animation could not be found", meshNames[i]); - return false; - } + // for(int i = 0; i < numFrames * numBones * 7; i++) { + // Log("Frame data @ %d is %f", i, frameData[i]); + // } + + asset.mBoneAnimationBuffer.mFrameLengthInMs = frameLengthInMs; + asset.mBoneAnimationBuffer.mNumFrames = numFrames; - boneAnimationData.mMeshTarget = entity; + asset.mBoneAnimationBuffer.mMeshTarget = entity; + auto& animation = asset.mAnimations[asset.mAnimations.size() - 1]; + animation.mStart = std::chrono::high_resolution_clock::now(); + animation.mAnimating = true; + animation.mReverse = false; + animation.mDuration = (frameLengthInMs * numFrames) / 1000.0f; + asset.mAnimating = true; - boneAnimationData.mFrameData.insert( - boneAnimationData.mFrameData.begin(), - frameData[i * numFrames * 7 * sizeof(float)], // 7 == x, y, z, w + three euler angles - frameData[(i+1) * numFrames * 7 * sizeof(float)] - ); + // // Log(", set start to %f and duration to %f", ); + // Log("Successfully set bone animation buffer, set start to %d, dur is %f", + // std::chrono::duration_cast(asset.mAnimations[1].mStart.time_since_epoch()).count(), + // asset.mAnimations[1].mDuration + // ); - asset.mBoneAnimationBuffer.mAnimations.push_back(boneAnimationData); - } return true; } -void AssetManager::setBoneTransform( - FilamentInstance* filamentInstance, - vector animations, - int frameNumber) { - RenderableManager &rm = _engine->getRenderableManager(); - - TransformManager &transformManager = _engine->getTransformManager(); - - auto frameDataOffset = frameNumber * 7; - - int skinIndex = 0; - - for(auto& animation : animations) { - - math::mat4f inverseGlobalTransform = inverse( - transformManager.getWorldTransform( - transformManager.getInstance(animation.mMeshTarget) - ) - ); - - utils::Entity joint = filamentInstance->getJointsAt(skinIndex)[animation.mBoneIndex]; - - math::mat4f localTransform(math::quatf{ - animation.mFrameData[frameDataOffset+6], - animation.mFrameData[frameDataOffset+3], - animation.mFrameData[frameDataOffset+4], - animation.mFrameData[frameDataOffset+5] - }); - - const math::mat4f& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(animation.skinIndex)[animation.mBoneIndex]; - auto jointInstance = transformManager.getInstance(joint); - math::mat4f globalJointTransform = transformManager.getWorldTransform(jointInstance); - - math::mat4f boneTransform = inverseGlobalTransform * globalJointTransform * localTransform * inverseBindMatrix; - auto renderable = rm.getInstance(animation.mMeshTarget); - rm.setBones( - renderable, - &boneTransform, - 1, - animation.mBoneIndex - ); - } -} void AssetManager::playAnimation(EntityId e, int index, bool loop, bool reverse) { const auto& pos = _entityIdLookup.find(e); @@ -527,11 +575,12 @@ void AssetManager::playAnimation(EntityId e, int index, bool loop, bool reverse) return; } auto& asset = _assets[pos->second]; - - asset.mAnimations[index+2].mAnimating = true; - asset.mAnimations[index+2].mStart = std::chrono::high_resolution_clock::now(); - asset.mAnimations[index+2].mLoop = loop; - asset.mAnimations[index+2].mReverse = reverse; + Log("Playing animation at %d", index); + + asset.mAnimations[index].mStart = std::chrono::high_resolution_clock::now(); + asset.mAnimations[index].mLoop = loop; + asset.mAnimations[index].mReverse = reverse; + asset.mAnimations[index].mAnimating = true; // Log("new start %d, dur is %f", std::chrono::duration_cast(asset.mAnimations[index+2].mStart.time_since_epoch()).count(), asset.mAnimations[index+2].mDuration); asset.mAnimating = true; } @@ -543,7 +592,7 @@ void AssetManager::stopAnimation(EntityId entityId, int index) { return; } auto& asset = _assets[pos->second]; - asset.mAnimations[index+2].mStart = time_point_t::max(); + asset.mAnimations[index].mStart = time_point_t::max(); } void AssetManager::loadTexture(EntityId entity, const char* resourcePath, int renderableIndex) { @@ -702,7 +751,7 @@ void AssetManager::transformToUnitCube(EntityId entity) { tm.setTransform(tm.getInstance(inst->getRoot()), transform); } -void AssetManager::updateTransform(SceneAsset asset) { +void AssetManager::updateTransform(SceneAsset& asset) { auto &tm = _engine->getTransformManager(); auto transform = asset.mPosition * asset.mRotation * math::mat4f::scaling(asset.mScale); diff --git a/ios/src/PolyvoxFilamentApi.cpp b/ios/src/PolyvoxFilamentApi.cpp index 265e4ca5..be0cdff5 100644 --- a/ios/src/PolyvoxFilamentApi.cpp +++ b/ios/src/PolyvoxFilamentApi.cpp @@ -358,20 +358,20 @@ extern "C" { FLUTTER_PLUGIN_EXPORT void set_bone_animation( void* assetManager, EntityId asset, - int length, - const char** const boneNames, - const char** const meshNames, const float* const frameData, int numFrames, + int numBones, + const char** const boneNames, + const char* const meshName, float frameLengthInMs) { //std::packaged_task lambda([=]() mutable { ((AssetManager*)assetManager)->setBoneAnimationBuffer( asset, - length, - boneNames, - meshNames, frameData, - numFrames, + numFrames, + numBones, + boneNames, + meshName, frameLengthInMs ); //}); @@ -417,7 +417,6 @@ extern "C" { bool reverse) { //std::packaged_task lambda([=]() mutable { - std::cout << "Playing animation" << std::endl; ((AssetManager*)assetManager)->playAnimation(asset, index, loop, reverse); //}); // auto fut = _tp->add_task(lambda); diff --git a/lib/animations/animation_builder.dart b/lib/animations/animation_builder.dart index 45455cac..15bd1df2 100644 --- a/lib/animations/animation_builder.dart +++ b/lib/animations/animation_builder.dart @@ -7,7 +7,7 @@ import 'package:vector_math/vector_math.dart'; class AnimationBuilder { final FilamentController controller; - BoneAnimationData? BoneAnimationData; + // BoneAnimationData? BoneAnimationData; double _frameLengthInMs = 0; double _duration = 0; @@ -16,7 +16,7 @@ class AnimationBuilder { double? _interpMorphStartValue; double? _interpMorphEndValue; - List? _BoneAnimationDatas = null; + // List? _BoneAnimationDatas = null; FilamentEntity asset; String meshName; diff --git a/lib/animations/bone_animation_data.dart b/lib/animations/bone_animation_data.dart index cb694b01..7bf9b7e3 100644 --- a/lib/animations/bone_animation_data.dart +++ b/lib/animations/bone_animation_data.dart @@ -5,6 +5,7 @@ import 'package:vector_math/vector_math.dart'; /// Model class for bone animation frame data. /// To create dynamic/runtime bone animations (as distinct from animations embedded in a glTF asset), create an instance containing the relevant /// data and pass to the [setBoneAnimation] method on a [FilamentController]. +/// [frameData] is laid out as [locX, locY, locZ, rotW, rotX, rotY, rotZ] /// class BoneAnimationData { final String boneName; diff --git a/lib/animations/bone_driver.dart b/lib/animations/bone_driver.dart index eeb0237a..e404efca 100644 --- a/lib/animations/bone_driver.dart +++ b/lib/animations/bone_driver.dart @@ -13,45 +13,64 @@ import 'package:vector_math/vector_math.dart'; /// 4) min/max rotation values (corresponding to -1/1 on the blendshape) /// +class Transformation { + final Quaternion rotation; + late final Vector3 translation; + + Transformation(this.rotation, {Vector3? translation}) { + this.translation = translation ?? Vector3.zero(); + } +} + class BoneDriver { final String bone; - final String blendshape; + final Map + transformations; // maps a blendshape key to a Transformation - late final Vector3 transMin; - late final Vector3 transMax; - late final Quaternion rotMin; - late final Quaternion rotMax; - - BoneDriver(this.bone, this.blendshape, this.rotMin, this.rotMax, - Vector3? transMin, Vector3? transMax) { - this.transMin = transMin ?? Vector3.zero(); - this.transMax = transMax ?? Vector3.zero(); - } - - factory BoneDriver.fromJsonObject(dynamic jsonObject) { - return BoneDriver( - jsonObject["bone"], - jsonObject["blendshape"], - Quaternion.fromFloat32List(Float32List.fromList(jsonObject["rotMin"])), - Quaternion.fromFloat32List(Float32List.fromList(jsonObject["rotMax"])), - Vector3.fromFloat32List(Float32List.fromList(jsonObject["transMin"])), - Vector3.fromFloat32List(Float32List.fromList(jsonObject["transMax"])), - ); - } + BoneDriver(this.bone, this.transformations); // // Accepts a Float32List containing [numFrames] frames of data for a single morph target weight (for efficiency, this must be unravelled to a single contiguous Float32List). // Returns a generator that yields [numFrames] Quaternions, each representing the (weighted) rotation/translation specified by the mapping of this BoneDriver. // - Iterable transform(List morphTargetFrameData) sync* { - for (int i = 0; i < morphTargetFrameData.length; i++) { - var weight = (morphTargetFrameData[i] / 2) + 0.5; + Iterable transform( + Map> morphTargetFrameData) sync* { + assert(setEquals( + morphTargetFrameData.keys.toSet(), transformations.keys.toSet())); + var numFrames = morphTargetFrameData.values.first.length; + assert(morphTargetFrameData.values.every((x) => x.length == numFrames)); + for (int frameNum = 0; frameNum < numFrames; frameNum++) { + var rotations = transformations.keys.map((blendshape) { + var weight = morphTargetFrameData[blendshape]![frameNum]; + var rotation = transformations[blendshape]!.rotation.clone(); + rotation.x *= weight; + rotation.y *= weight; + rotation.z *= weight; + return rotation; + }).toList(); - yield Quaternion( - rotMin.x + (weight * (rotMax.x - rotMin.x)), - rotMin.y + (weight * (rotMax.y - rotMin.y)), - rotMin.z + (weight * (rotMax.z - rotMin.z)), - 1.0); + yield rotations.fold( + rotations.first, (Quaternion a, Quaternion b) => a * b); + // todo - bone translations } } + + factory BoneDriver.fromJsonObject(dynamic jsonObject) { + throw Exception("TODO"); + // return BoneDriver( + // jsonObject["bone"], + // Map.fromIterable(jsonObject["blendshape"].map((bsName, quats) { + // var q = quats.map(()) + // MapEntry(k, + } } + + + + + // } + // yield Quaternion( + // rotMin.x + (weight * (rotMax.x - rotMin.x)), + // rotMin.y + (weight * (rotMax.y - rotMin.y)), + // rotMin.z + (weight * (rotMax.z - rotMin.z)), + // 1.0); \ No newline at end of file diff --git a/lib/animations/csv_animation.dart b/lib/animations/csv_animation.dart index b46d35d5..1b9182e5 100644 --- a/lib/animations/csv_animation.dart +++ b/lib/animations/csv_animation.dart @@ -15,40 +15,52 @@ class DynamicAnimation { final List boneAnimation; factory DynamicAnimation.load(String meshName, String csvPath, - {String? boneDriverConfigPath}) { + {List? boneDrivers, + String? boneDriverConfigPath, + double? framerate}) { // create a MorphAnimationData instance from the given CSV var llf = _loadLiveLinkFaceCSV(csvPath); + var frameLengthInMs = 1000 / (framerate ?? 60.0); var morphNames = llf .item1; //.where((name) => !boneDrivers.any((element) => element.blendshape == name)); - var morphAnimationData = MorphAnimationData( - meshName, - llf.item2, - morphNames, - 1000 / 60.0, - ); + var morphAnimationData = + MorphAnimationData(meshName, llf.item2, morphNames, frameLengthInMs); final boneAnimations = []; // if applicable, load the bone driver config if (boneDriverConfigPath != null) { - var boneData = json.decode(File(boneDriverConfigPath).readAsStringSync()); - // for each driver - for (var key in boneData.keys()) { - var driver = BoneDriver.fromJsonObject(boneData[key]); + if (boneDrivers != null) { + throw Exception( + "Specify either boneDrivers, or the config path, not both"); + } + boneDrivers = [ + json + .decode(File(boneDriverConfigPath).readAsStringSync()) + .map(BoneDriver.fromJsonObject) + .toList() + ]; + } + // iterate over every bone driver + if (boneDrivers != null) { + for (var driver in boneDrivers) { // get all frames for the single the blendshape - var morphFrameData = - morphAnimationData.getData(driver.blendshape).toList(); + var morphData = driver.transformations + .map((String blendshape, Transformation transformation) { + return MapEntry( + blendshape, morphAnimationData.getData(blendshape).toList()); + }); // apply the driver to the blendshape weight - var transformedQ = driver.transform(morphFrameData).toList(); + var transformedQ = driver.transform(morphData).toList(); // transform the quaternion to a Float32List var transformedF = _quaternionToFloatList(transformedQ); // add to the list of boneAnimations boneAnimations.add(BoneAnimationData( - driver.bone, meshName, transformedF, 1000.0 / 60.0)); + driver.bone, meshName, transformedF, frameLengthInMs)); } } @@ -56,9 +68,11 @@ class DynamicAnimation { } static Float32List _quaternionToFloatList(List quats) { - var data = Float32List(quats.length * 4); + var data = Float32List(quats.length * 7); + int i = 0; for (var quat in quats) { - data.addAll([0, 0, 0, quat.w, quat.x, quat.y, quat.z]); + data.setRange(i, i + 7, [0, 0, 0, quat.w, quat.x, quat.y, quat.z]); + i += 7; } return data; } @@ -86,7 +100,7 @@ class DynamicAnimation { // CSVs may contain rows where the "BlendShapeCount" column is set to "0" and/or the weight columns are simply missing. // This can happen when something went wrong while recording via an app (e.g. LiveLinkFace) // Whenever we encounter this type of row, we consider that all weights should be set to zero for that frame. - if (numFrameWeights == int.parse(frame[1])) { + if (numFrameWeights != int.parse(frame[1])) { _data.addAll(List.filled(numBlendShapes, 0.0)); continue; } diff --git a/lib/animations/live_link_face_bone_driver.dart b/lib/animations/live_link_face_bone_driver.dart new file mode 100644 index 00000000..b5cc9c99 --- /dev/null +++ b/lib/animations/live_link_face_bone_driver.dart @@ -0,0 +1,12 @@ +import 'dart:math'; + +import 'package:polyvox_filament/animations/bone_driver.dart'; +import 'package:vector_math/vector_math.dart'; + +BoneDriver getLiveLinkFaceBoneDrivers(String bone) { + return BoneDriver(bone, { + "HeadPitch": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)), + "HeadRoll": Transformation(Quaternion.axisAngle(Vector3(0, 0, 1), pi / 2)), + "HeadYaw": Transformation(Quaternion.axisAngle(Vector3(0, 1, 0), pi / 2)), + }); +} diff --git a/lib/animations/morph_animation_data.dart b/lib/animations/morph_animation_data.dart index ac6ab4a2..ad7ce936 100644 --- a/lib/animations/morph_animation_data.dart +++ b/lib/animations/morph_animation_data.dart @@ -24,8 +24,9 @@ class MorphAnimationData { final double frameLengthInMs; Iterable getData(String morphName) sync* { + int index = morphNames.indexOf(morphName); for (int i = 0; i < numFrames; i++) { - yield data[i * numMorphWeights]; + yield data[(i * numMorphWeights) + index]; } } } diff --git a/lib/filament_controller.dart b/lib/filament_controller.dart index 1e6c6dad..52df302b 100644 --- a/lib/filament_controller.dart +++ b/lib/filament_controller.dart @@ -323,15 +323,19 @@ class FilamentController { /// void setBoneAnimation( FilamentEntity asset, List animations) async { + // for future compatibility, instances of BoneAnimationData can specify individual mesh targets + // however on the rendering side we currently only allow one set of frame data for one mesh target (though multiple bones are supported). + // this is a check that all animations are targeting the same mesh + assert(animations.map((e) => e.meshName).toSet().length == 1); + var data = calloc(animations.length * animations.first.frameData.length); int offset = 0; - var numFrames = animations.first.frameData.length; - var meshNames = calloc>(animations.length); + var numFrames = animations.first.frameData.length ~/ 7; var boneNames = calloc>(animations.length); int animIdx = 0; for (var animation in animations) { - if (animation.frameData.length != numFrames) { + if (animation.frameData.length ~/ 7 != numFrames) { throw Exception( "All bone animations must share the same animation frame data length."); } @@ -339,20 +343,19 @@ class FilamentController { data.elementAt(offset).value = animation.frameData[i]; offset += 1; } - meshNames.elementAt(animIdx).value = - animation.meshName.toNativeUtf8().cast(); boneNames.elementAt(animIdx).value = animation.boneName.toNativeUtf8().cast(); + animIdx++; } _nativeLibrary.set_bone_animation( _assetManager, asset, - animations.length, - boneNames, - meshNames, data, numFrames, + animations.length, + boneNames, + animations.first.meshName.toNativeUtf8().cast(), animations.first.frameLengthInMs); calloc.free(data); } @@ -379,7 +382,6 @@ class FilamentController { void playAnimation(FilamentEntity asset, int index, {bool loop = false, bool reverse = false}) async { - print("LOOP $loop"); _nativeLibrary.play_animation( _assetManager, asset, index, loop ? 1 : 0, reverse ? 1 : 0); } diff --git a/lib/generated_bindings.dart b/lib/generated_bindings.dart index fbba1e36..2197eb07 100644 --- a/lib/generated_bindings.dart +++ b/lib/generated_bindings.dart @@ -655,21 +655,21 @@ class NativeLibrary { void set_bone_animation( ffi.Pointer assetManager, int asset, - int length, - ffi.Pointer> boneNames, - ffi.Pointer> meshNames, ffi.Pointer frameData, int numFrames, + int numBones, + ffi.Pointer> boneNames, + ffi.Pointer meshName, double frameLengthInMs, ) { return _set_bone_animation( assetManager, asset, - length, - boneNames, - meshNames, frameData, numFrames, + numBones, + boneNames, + meshName, frameLengthInMs, ); } @@ -679,21 +679,21 @@ class NativeLibrary { ffi.Void Function( ffi.Pointer, EntityId, - ffi.Int, - ffi.Pointer>, - ffi.Pointer>, ffi.Pointer, ffi.Int, + ffi.Int, + ffi.Pointer>, + ffi.Pointer, ffi.Float)>>('set_bone_animation'); late final _set_bone_animation = _set_bone_animationPtr.asFunction< void Function( ffi.Pointer, int, - int, - ffi.Pointer>, - ffi.Pointer>, ffi.Pointer, int, + int, + ffi.Pointer>, + ffi.Pointer, double)>(); void play_animation( @@ -1122,96 +1122,9 @@ class NativeLibrary { late final _ios_dummy = _ios_dummyPtr.asFunction(); } -class __mbstate_t extends ffi.Union { - @ffi.Array.multi([128]) - external ffi.Array __mbstate8; - - @ffi.LongLong() - external int _mbstateL; -} - -class __darwin_pthread_handler_rec extends ffi.Struct { - external ffi - .Pointer)>> - __routine; - - external ffi.Pointer __arg; - - external ffi.Pointer<__darwin_pthread_handler_rec> __next; -} - -class _opaque_pthread_attr_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([56]) - external ffi.Array __opaque; -} - -class _opaque_pthread_cond_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([40]) - external ffi.Array __opaque; -} - -class _opaque_pthread_condattr_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([8]) - external ffi.Array __opaque; -} - -class _opaque_pthread_mutex_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([56]) - external ffi.Array __opaque; -} - -class _opaque_pthread_mutexattr_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([8]) - external ffi.Array __opaque; -} - -class _opaque_pthread_once_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([8]) - external ffi.Array __opaque; -} - -class _opaque_pthread_rwlock_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([192]) - external ffi.Array __opaque; -} - -class _opaque_pthread_rwlockattr_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([16]) - external ffi.Array __opaque; -} - -class _opaque_pthread_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; - - @ffi.Array.multi([8176]) - external ffi.Array __opaque; +class __fsid_t extends ffi.Struct { + @ffi.Array.multi([2]) + external ffi.Array __val; } class ResourceBuffer extends ffi.Struct { @@ -1225,8 +1138,6 @@ class ResourceBuffer extends ffi.Struct { } class ResourceLoaderWrapper extends ffi.Struct { - external ffi.Pointer mOwner; - external LoadResource mLoadResource; external FreeResource mFreeResource; @@ -1234,6 +1145,8 @@ class ResourceLoaderWrapper extends ffi.Struct { external LoadResourceFromOwner mLoadResourceFromOwner; external FreeResourceFromOwner mFreeResourceFromOwner; + + external ffi.Pointer mOwner; } typedef LoadResource = ffi.Pointer< @@ -1248,75 +1161,127 @@ typedef FreeResourceFromOwner = ffi.Pointer< ffi.Void Function(ResourceBuffer, ffi.Pointer)>>; typedef EntityId = ffi.Int32; +const int _STDINT_H = 1; + +const int _FEATURES_H = 1; + +const int _DEFAULT_SOURCE = 1; + +const int __GLIBC_USE_ISOC2X = 1; + +const int __USE_ISOC11 = 1; + +const int __USE_ISOC99 = 1; + +const int __USE_ISOC95 = 1; + +const int _POSIX_SOURCE = 1; + +const int _POSIX_C_SOURCE = 200809; + +const int __USE_POSIX = 1; + +const int __USE_POSIX2 = 1; + +const int __USE_POSIX199309 = 1; + +const int __USE_POSIX199506 = 1; + +const int __USE_XOPEN2K = 1; + +const int __USE_XOPEN2K8 = 1; + +const int _ATFILE_SOURCE = 1; + const int __WORDSIZE = 64; -const int __DARWIN_ONLY_64_BIT_INO_T = 1; +const int __WORDSIZE_TIME64_COMPAT32 = 1; -const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; +const int __SYSCALL_WORDSIZE = 64; -const int __DARWIN_ONLY_VERS_1050 = 1; +const int __TIMESIZE = 64; -const int __DARWIN_UNIX03 = 1; +const int __USE_MISC = 1; -const int __DARWIN_64_BIT_INO_T = 1; +const int __USE_ATFILE = 1; -const int __DARWIN_VERS_1050 = 1; +const int __USE_FORTIFY_LEVEL = 0; -const int __DARWIN_NON_CANCELABLE = 0; +const int __GLIBC_USE_DEPRECATED_GETS = 0; -const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; +const int __GLIBC_USE_DEPRECATED_SCANF = 0; -const int __DARWIN_C_ANSI = 4096; +const int _STDC_PREDEF_H = 1; -const int __DARWIN_C_FULL = 900000; +const int __STDC_IEC_559__ = 1; -const int __DARWIN_C_LEVEL = 900000; +const int __STDC_IEC_60559_BFP__ = 201404; -const int __STDC_WANT_LIB_EXT1__ = 1; +const int __STDC_IEC_559_COMPLEX__ = 1; -const int __DARWIN_NO_LONG_LONG = 0; +const int __STDC_IEC_60559_COMPLEX__ = 201404; -const int _DARWIN_FEATURE_64_BIT_INODE = 1; +const int __STDC_ISO_10646__ = 201706; -const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; +const int __GNU_LIBRARY__ = 6; -const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; +const int __GLIBC__ = 2; -const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; +const int __GLIBC_MINOR__ = 37; -const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; +const int _SYS_CDEFS_H = 1; -const int __has_ptrcheck = 0; +const int __THROW = 1; -const int __DARWIN_NULL = 0; +const int __THROWNL = 1; -const int __PTHREAD_SIZE__ = 8176; +const int __glibc_c99_flexarr_available = 1; -const int __PTHREAD_ATTR_SIZE__ = 56; +const int __LDOUBLE_REDIRECTS_TO_FLOAT128_ABI = 0; -const int __PTHREAD_MUTEXATTR_SIZE__ = 8; +const int __HAVE_GENERIC_SELECTION = 0; -const int __PTHREAD_MUTEX_SIZE__ = 56; +const int __GLIBC_USE_LIB_EXT2 = 1; -const int __PTHREAD_CONDATTR_SIZE__ = 8; +const int __GLIBC_USE_IEC_60559_BFP_EXT = 1; -const int __PTHREAD_COND_SIZE__ = 40; +const int __GLIBC_USE_IEC_60559_BFP_EXT_C2X = 1; -const int __PTHREAD_ONCE_SIZE__ = 8; +const int __GLIBC_USE_IEC_60559_EXT = 1; -const int __PTHREAD_RWLOCK_SIZE__ = 192; +const int __GLIBC_USE_IEC_60559_FUNCS_EXT = 1; -const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; +const int __GLIBC_USE_IEC_60559_FUNCS_EXT_C2X = 1; -const int USER_ADDR_NULL = 0; +const int __GLIBC_USE_IEC_60559_TYPES_EXT = 1; -const int INT8_MAX = 127; +const int _BITS_TYPES_H = 1; -const int INT16_MAX = 32767; +const int _BITS_TYPESIZES_H = 1; -const int INT32_MAX = 2147483647; +const int __OFF_T_MATCHES_OFF64_T = 1; -const int INT64_MAX = 9223372036854775807; +const int __INO_T_MATCHES_INO64_T = 1; + +const int __RLIM_T_MATCHES_RLIM64_T = 1; + +const int __STATFS_MATCHES_STATFS64 = 1; + +const int __KERNEL_OLD_TIMEVAL_MATCHES_TIMEVAL64 = 1; + +const int __FD_SETSIZE = 1024; + +const int _BITS_TIME64_H = 1; + +const int _BITS_WCHAR_H = 1; + +const int __WCHAR_MAX = 2147483647; + +const int __WCHAR_MIN = -2147483648; + +const int _BITS_STDINT_INTN_H = 1; + +const int _BITS_STDINT_UINTN_H = 1; const int INT8_MIN = -128; @@ -1326,6 +1291,14 @@ const int INT32_MIN = -2147483648; const int INT64_MIN = -9223372036854775808; +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + const int UINT8_MAX = 255; const int UINT16_MAX = 65535; @@ -1360,66 +1333,54 @@ const int UINT_LEAST64_MAX = -1; const int INT_FAST8_MIN = -128; -const int INT_FAST16_MIN = -32768; +const int INT_FAST16_MIN = -9223372036854775808; -const int INT_FAST32_MIN = -2147483648; +const int INT_FAST32_MIN = -9223372036854775808; const int INT_FAST64_MIN = -9223372036854775808; const int INT_FAST8_MAX = 127; -const int INT_FAST16_MAX = 32767; +const int INT_FAST16_MAX = 9223372036854775807; -const int INT_FAST32_MAX = 2147483647; +const int INT_FAST32_MAX = 9223372036854775807; const int INT_FAST64_MAX = 9223372036854775807; const int UINT_FAST8_MAX = 255; -const int UINT_FAST16_MAX = 65535; +const int UINT_FAST16_MAX = -1; -const int UINT_FAST32_MAX = 4294967295; +const int UINT_FAST32_MAX = -1; const int UINT_FAST64_MAX = -1; -const int INTPTR_MAX = 9223372036854775807; - const int INTPTR_MIN = -9223372036854775808; +const int INTPTR_MAX = 9223372036854775807; + const int UINTPTR_MAX = -1; +const int INTMAX_MIN = -9223372036854775808; + const int INTMAX_MAX = 9223372036854775807; const int UINTMAX_MAX = -1; -const int INTMAX_MIN = -9223372036854775808; - const int PTRDIFF_MIN = -9223372036854775808; const int PTRDIFF_MAX = 9223372036854775807; -const int SIZE_MAX = -1; - -const int RSIZE_MAX = 9223372036854775807; - -const int WCHAR_MAX = 2147483647; - -const int WCHAR_MIN = -2147483648; - -const int WINT_MIN = -2147483648; - -const int WINT_MAX = 2147483647; - const int SIG_ATOMIC_MIN = -2147483648; const int SIG_ATOMIC_MAX = 2147483647; -const int __DARWIN_WCHAR_MAX = 2147483647; +const int SIZE_MAX = -1; -const int __DARWIN_WCHAR_MIN = -2147483648; +const int WCHAR_MIN = -2147483648; -const int __DARWIN_WEOF = -1; +const int WCHAR_MAX = 2147483647; -const int _FORTIFY_SOURCE = 2; +const int WINT_MIN = 0; -const int NULL = 0; +const int WINT_MAX = 4294967295; diff --git a/test/bone_driver_test.dart b/test/bone_driver_test.dart index 09a90305..1119876f 100644 --- a/test/bone_driver_test.dart +++ b/test/bone_driver_test.dart @@ -1,73 +1,70 @@ -import 'dart:convert'; -import 'dart:math'; -import 'dart:typed_data'; import 'package:flutter_test/flutter_test.dart'; import 'package:polyvox_filament/animations/bone_driver.dart'; import 'package:vector_math/vector_math.dart'; void main() { group('BoneDriver', () { - test('constructor sets correct values', () { - Quaternion rotMin = Quaternion.identity(); - Quaternion rotMax = Quaternion.axisAngle(Vector3(1, 0, 0), 0.5); - Vector3 transMin = Vector3.zero(); - Vector3 transMax = Vector3(1, 1, 1); - - BoneDriver boneDriver = BoneDriver( - 'bone1', 'blendshape1', rotMin, rotMax, transMin, transMax); - - expect(boneDriver.bone, 'bone1'); - expect(boneDriver.blendshape, 'blendshape1'); - expect(boneDriver.rotMin, rotMin); - expect(boneDriver.rotMax, rotMax); - expect(boneDriver.transMin, transMin); - expect(boneDriver.transMax, transMax); - }); - - test('fromJsonObject creates BoneDriver instance correctly', () { - dynamic jsonObject = { - "bone": "bone1", - "blendshape": "blendshape1", - "rotMin": Quaternion.identity().storage, - "rotMax": Quaternion.axisAngle(Vector3(1, 0, 0), 0.5).storage, - "transMin": Vector3.zero().storage, - "transMax": Vector3(1, 1, 1).storage + test( + 'transform should yield correct Quaternions for given morphTargetFrameData', + () { + final bone = 'bone1'; + final transformations = { + 'blendshape1': Transformation(Quaternion(1, 0, 0, 1)), + 'blendshape2': Transformation(Quaternion(0, 1, 0, 1)), }; + final morphTargetFrameData = >{ + 'blendshape1': [0.5, -0.5], + 'blendshape2': [-1, 1], + }; + final boneDriver = BoneDriver(bone, transformations); - BoneDriver boneDriver = BoneDriver.fromJsonObject(jsonObject); + final result = boneDriver.transform(morphTargetFrameData).toList(); - expect(boneDriver.bone, 'bone1'); - expect(boneDriver.blendshape, 'blendshape1'); - expect(boneDriver.rotMin.absoluteError(Quaternion.identity()), 0); - expect( - boneDriver.rotMax - .absoluteError(Quaternion.axisAngle(Vector3(1, 0, 0), 0.5)), - 0); - expect(boneDriver.transMin.absoluteError(Vector3.zero()), 0); - expect(boneDriver.transMax.absoluteError(Vector3(1, 1, 1)), 0); + expect(result.length, 2); + expect(result[0].x, -0.5); + expect(result[0].y, 0); + expect(result[0].z, -0.5); + expect(result[0].w, 0); + expect(result[1].x, 0.5); + expect(result[1].y, 0); + expect(result[1].z, 0.5); + expect(result[1].w, 0); }); - test('transform generates correct Quaternions', () { - Quaternion rotMin = Quaternion.identity(); - Quaternion rotMax = Quaternion.axisAngle(Vector3(1, 0, 0), 0.5); - BoneDriver boneDriver = - BoneDriver('bone1', 'blendshape1', rotMin, rotMax, null, null); + test( + 'transform should throw AssertionError when morphTargetFrameData keys do not match transformations keys', + () { + final bone = 'bone1'; + final transformations = { + 'blendshape1': Transformation(Quaternion(1, 0, 0, 0)), + 'blendshape2': Transformation(Quaternion(0, 1, 0, 0)), + }; + final morphTargetFrameData = >{ + 'blendshape1': [0.5, -0.5], + 'blendshape3': [-1, 1], + }; + final boneDriver = BoneDriver(bone, transformations); - List morphTargetFrameData = [-1, 0, 1]; - List expectedResult = [ - Quaternion(rotMin.x, rotMin.y, rotMin.z, 1.0), - Quaternion((rotMin.x + rotMax.x) / 2, (rotMin.y + rotMax.y) / 2, - (rotMin.z + rotMax.z) / 2, 1.0), - Quaternion(rotMax.x, rotMax.y, rotMax.z, 1.0), - ]; + expect(() => boneDriver.transform(morphTargetFrameData), + throwsA(isA())); + }); - Iterable result = boneDriver.transform(morphTargetFrameData); - List resultAsList = result.toList(); - expect(resultAsList.length, expectedResult.length); + test( + 'transform should throw AssertionError when morphTargetFrameData values lengths do not match', + () { + final bone = 'bone1'; + final transformations = { + 'blendshape1': Transformation(Quaternion(1, 0, 0, 0)), + 'blendshape2': Transformation(Quaternion(0, 1, 0, 0)), + }; + final morphTargetFrameData = >{ + 'blendshape1': [0.5, -0.5], + 'blendshape2': [-1], + }; + final boneDriver = BoneDriver(bone, transformations); - for (int i = 0; i < expectedResult.length; i++) { - expect(resultAsList[i].absoluteError(expectedResult[i]), 0); - } + expect(() => boneDriver.transform(morphTargetFrameData), + throwsA(isA())); }); }); } From bbb3cb5727da660d0ce88b25cb7cb4d06f12aa0f Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Fri, 28 Apr 2023 20:53:16 +0800 Subject: [PATCH 8/8] fix dynamic bone animations --- example/lib/main.dart | 7 +- ios/include/AssetManager.hpp | 3 +- ios/include/PolyvoxFilamentApi.h | 3 +- ios/include/SceneAsset.hpp | 15 +- ios/src/AssetManager.cpp | 168 ++++++++++-------- ios/src/FilamentViewer.cpp | 2 +- ios/src/PolyvoxFilamentApi.cpp | 6 +- lib/animations/bone_animation_data.dart | 4 +- lib/animations/bone_driver.dart | 14 +- lib/animations/csv_animation.dart | 16 +- .../live_link_face_bone_driver.dart | 5 +- lib/filament_controller.dart | 47 +++-- lib/generated_bindings.dart | 10 +- 13 files changed, 170 insertions(+), 130 deletions(-) diff --git a/example/lib/main.dart b/example/lib/main.dart index db424975..bad4a0a2 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -214,9 +214,10 @@ class _MyAppState extends State with SingleTickerProviderStateMixin { return vals; }).toList()); - _filamentController.setBoneAnimation(_cube!, [ - BoneAnimationData("Bone.001", "Cube.001", frameData, 1000.0 / 60.0) - ]); + _filamentController.setBoneAnimation( + _cube!, + BoneAnimationData( + "Bone.001", ["Cube.001"], frameData, 1000.0 / 60.0)); // , // "Bone.001", // "Cube.001", diff --git a/ios/include/AssetManager.hpp b/ios/include/AssetManager.hpp index 7b2da476..73333b6b 100644 --- a/ios/include/AssetManager.hpp +++ b/ios/include/AssetManager.hpp @@ -54,7 +54,8 @@ namespace polyvox { int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshName, + int numMeshTargets, float frameLengthInMs); void playAnimation(EntityId e, int index, bool loop, bool reverse); void stopAnimation(EntityId e, int index); diff --git a/ios/include/PolyvoxFilamentApi.h b/ios/include/PolyvoxFilamentApi.h index 8f92756f..2739f4e8 100644 --- a/ios/include/PolyvoxFilamentApi.h +++ b/ios/include/PolyvoxFilamentApi.h @@ -64,7 +64,8 @@ void set_bone_animation( int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshName, + int numMeshTargets, float frameLengthInMs); void play_animation(void* assetManager, EntityId asset, int index, bool loop, bool reverse); diff --git a/ios/include/SceneAsset.hpp b/ios/include/SceneAsset.hpp index 1eb93924..d7ee58e5 100644 --- a/ios/include/SceneAsset.hpp +++ b/ios/include/SceneAsset.hpp @@ -22,7 +22,7 @@ extern "C" { #include "PolyvoxFilamentApi.h" } - +template class std::vector; namespace polyvox { using namespace filament; using namespace filament::gltfio; @@ -56,8 +56,12 @@ namespace polyvox { // Multiple bones are supported but these must be skinned to a single mesh target. // struct BoneAnimationBuffer { - utils::Entity mMeshTarget; + vector mMeshTargets; vector mBones; + vector mBaseTransforms; + // vector mBaseTranslations; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this. + // vector mBaseRotations; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this. + // vector mBaseScales; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this. size_t skinIndex = 0; int mNumFrames = -1; float mFrameLengthInMs = 0; @@ -99,10 +103,5 @@ namespace polyvox { mAnimations[i].mDuration = mAnimator->getAnimationDuration(i); } } - - }; - -} - - \ No newline at end of file +} \ No newline at end of file diff --git a/ios/src/AssetManager.cpp b/ios/src/AssetManager.cpp index 5c1f83cb..1dcbc962 100644 --- a/ios/src/AssetManager.cpp +++ b/ios/src/AssetManager.cpp @@ -1,16 +1,17 @@ #include "AssetManager.hpp" -#include "Log.hpp" #include #include #include #include #include + #include #include #include #include #include +#include #include @@ -322,8 +323,10 @@ void AssetManager::updateAnimations() { frameNumber ); } + asset.mAnimator->updateBoneMatrices(); } } + } void AssetManager::setBoneTransform(SceneAsset& asset, int frameNumber) { @@ -336,61 +339,30 @@ void AssetManager::updateAnimations() { int skinIndex = 0; - math::mat4f inverseGlobalTransform = inverse( - transformManager.getWorldTransform( - transformManager.getInstance(asset.mBoneAnimationBuffer.mMeshTarget) - ) - ); - - auto renderable = rm.getInstance(asset.mBoneAnimationBuffer.mMeshTarget); - for(int i = 0; i < asset.mBoneAnimationBuffer.mBones.size(); i++) { auto mBoneIndex = asset.mBoneAnimationBuffer.mBones[i]; - auto frameDataOffset = (frameNumber * asset.mBoneAnimationBuffer.mBones.size() * 7) + asset.mBoneAnimationBuffer.mBones[i]; + auto frameDataOffset = (frameNumber * asset.mBoneAnimationBuffer.mBones.size() * 7) + (i * 7); utils::Entity joint = filamentInstance->getJointsAt(skinIndex)[mBoneIndex]; if(joint.isNull()) { Log("ERROR : joint not found"); continue; } - // RenderableManager::Bone bone { math::quatf{ - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5] - // }, - // math::float3 { - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+0], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+1], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+2] - // } - - // }; - // rm.setBones( - // renderable, - // &bone, - // 1, - // mBoneIndex - // ); - const math::mat4f localTransform(math::quatf{ - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3], - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4], - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5], - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6] + + vector& fd = asset.mBoneAnimationBuffer.mFrameData; + + math::mat4f localTransform(math::quatf { + fd[frameDataOffset+3], + fd[frameDataOffset+4], + fd[frameDataOffset+5], + fd[frameDataOffset+6], }); - const math::mat4f& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(skinIndex)[mBoneIndex]; + auto jointInstance = transformManager.getInstance(joint); - math::mat4f globalJointTransform = transformManager.getWorldTransform(jointInstance); - - math::mat4f boneTransform = inverseGlobalTransform * globalJointTransform * inverseBindMatrix * localTransform; - rm.setBones( - renderable, - &boneTransform, - 1, - mBoneIndex - ); + auto xform = asset.mBoneAnimationBuffer.mBaseTransforms[i]; + transformManager.setTransform(jointInstance, xform * localTransform); } } @@ -486,22 +458,18 @@ bool AssetManager::setBoneAnimationBuffer( int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshNames, + int numMeshTargets, float frameLengthInMs) { + + const auto& pos = _entityIdLookup.find(entityId); if(pos == _entityIdLookup.end()) { Log("ERROR: asset not found for entity."); return false; } auto& asset = _assets[pos->second]; - - auto entity = findEntityByName(asset, meshName); - if(!entity) { - Log("Mesh target %s for bone animation could not be found", meshName); - return false; - } - auto filamentInstance = asset.mAsset->getInstance(); size_t skinCount = filamentInstance->getSkinCount(); @@ -510,46 +478,71 @@ bool AssetManager::setBoneAnimationBuffer( Log("WARNING - skin count > 1 not currently implemented. This will probably not work"); } + TransformManager &transformManager = _engine->getTransformManager(); + int skinIndex = 0; const utils::Entity* joints = filamentInstance->getJointsAt(skinIndex); size_t numJoints = filamentInstance->getJointCountAt(skinIndex); - asset.mBoneAnimationBuffer.mBones.clear(); + BoneAnimationBuffer& animationBuffer = asset.mBoneAnimationBuffer; + + // if an animation has already been set, reset the transform for the respective bones + for(int i = 0; i < animationBuffer.mBones.size(); i++) { + auto boneIndex = animationBuffer.mBones[i]; + auto jointInstance = transformManager.getInstance(joints[boneIndex]); + transformManager.setTransform(jointInstance, animationBuffer.mBaseTransforms[i]); + } + + asset.mAnimator->resetBoneMatrices(); + + animationBuffer.mBones.resize(numBones); + animationBuffer.mBaseTransforms.resize(numBones); + for(int i = 0; i < numBones; i++) { Log("Bone %s", boneNames[i]); for(int j = 0; j < numJoints; j++) { const char* jointName = _ncm->getName(_ncm->getInstance(joints[j])); if(strcmp(jointName, boneNames[i]) == 0) { - asset.mBoneAnimationBuffer.mBones.push_back(j); + auto jointInstance = transformManager.getInstance(joints[j]); + // auto currentXform = ; + auto baseTransform = transformManager.getTransform(jointInstance); // inverse(filamentInstance->getInverseBindMatricesAt(skinIndex)[j]); + animationBuffer.mBaseTransforms[i] = baseTransform; + animationBuffer.mBones[i] = j; break; } } } - if(asset.mBoneAnimationBuffer.mBones.size() != numBones) { + if(animationBuffer.mBones.size() != numBones) { Log("Failed to find one or more bone indices"); return false; } - asset.mBoneAnimationBuffer.mFrameData.clear(); + animationBuffer.mFrameData.clear(); // 7 == locX, locY, locZ, rotW, rotX, rotY, rotZ - asset.mBoneAnimationBuffer.mFrameData.resize(numFrames * numBones * 7); - asset.mBoneAnimationBuffer.mFrameData.insert( - asset.mBoneAnimationBuffer.mFrameData.begin(), + animationBuffer.mFrameData.resize(numFrames * numBones * 7); + animationBuffer.mFrameData.insert( + animationBuffer.mFrameData.begin(), frameData, frameData + numFrames * numBones * 7 ); Log("%d frames for %d bones", numFrames, numBones); - // for(int i = 0; i < numFrames * numBones * 7; i++) { - // Log("Frame data @ %d is %f", i, frameData[i]); - // } - - asset.mBoneAnimationBuffer.mFrameLengthInMs = frameLengthInMs; - asset.mBoneAnimationBuffer.mNumFrames = numFrames; + animationBuffer.mFrameLengthInMs = frameLengthInMs; + animationBuffer.mNumFrames = numFrames; - asset.mBoneAnimationBuffer.mMeshTarget = entity; + animationBuffer.mMeshTargets.clear(); + for(int i = 0; i < numMeshTargets; i++) { + auto entity = findEntityByName(asset, meshNames[i]); + if(!entity) { + Log("Mesh target %s for bone animation could not be found", meshNames[i]); + return false; + } + Log("Added mesh target %s", meshNames[i]); + animationBuffer.mMeshTargets.push_back(entity); + } + auto& animation = asset.mAnimations[asset.mAnimations.size() - 1]; animation.mStart = std::chrono::high_resolution_clock::now(); animation.mAnimating = true; @@ -557,12 +550,6 @@ bool AssetManager::setBoneAnimationBuffer( animation.mDuration = (frameLengthInMs * numFrames) / 1000.0f; asset.mAnimating = true; - // // Log(", set start to %f and duration to %f", ); - // Log("Successfully set bone animation buffer, set start to %d, dur is %f", - // std::chrono::duration_cast(asset.mAnimations[1].mStart.time_since_epoch()).count(), - // asset.mAnimations[1].mDuration - // ); - return true; } @@ -833,3 +820,40 @@ size_t AssetManager::getLightEntityCount(EntityId entity) const noexcept { } // namespace polyvox + + +// auto& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(skinIndex)[mBoneIndex]; + + // auto globalJointTransform = transformManager.getWorldTransform(jointInstance); + + // for(auto& target : asset.mBoneAnimationBuffer.mMeshTargets) { + + // auto inverseGlobalTransform = inverse( + // transformManager.getWorldTransform( + // transformManager.getInstance(target) + // ) + // ); + + // auto boneTransform = inverseGlobalTransform * globalJointTransform * localTransform * inverseBindMatrix; + // auto renderable = rm.getInstance(target); + // rm.setBones( + // renderable, + // &boneTransform, + // 1, + // mBoneIndex + // ); + // } + + + + // 1.0f, 0.0f, 0.0f, 0.0f, + // 0.0f, 0.0f, 1.0f, 0.0f, + // 0.0f, -1.0f, 0.0f, 0.0f, + // 0.0f, 0.0f, 0.0f, 1.0f + // }; + // Log("TRANSFORM"); + // Log("%f %f %f %f", localTransform[0][0], localTransform[1][0], localTransform[2][0], localTransform[3][0] ) ; + // Log("%f %f %f %f", localTransform[0][1], localTransform[1][1], localTransform[2][1], localTransform[3][1] ) ; + // Log("%f %f %f %f", localTransform[0][2], localTransform[1][2], localTransform[2][2], localTransform[3][2] ) ; + // Log("%f %f %f %f", localTransform[0][3], localTransform[1][3], localTransform[2][3], localTransform[3][3] ) ; + // transformManager.getTransform(jointInstance); \ No newline at end of file diff --git a/ios/src/FilamentViewer.cpp b/ios/src/FilamentViewer.cpp index 6fdab983..b1e98329 100644 --- a/ios/src/FilamentViewer.cpp +++ b/ios/src/FilamentViewer.cpp @@ -788,7 +788,7 @@ void FilamentViewer::render(uint64_t frameTimeInNanos) { } if(_frameCount == 60) { - Log("1 sec average for asset animation update %f", _elapsed); + Log("1 sec average for asset animation update %f", _elapsed / 60); _elapsed = 0; _frameCount = 0; } diff --git a/ios/src/PolyvoxFilamentApi.cpp b/ios/src/PolyvoxFilamentApi.cpp index be0cdff5..6aa69400 100644 --- a/ios/src/PolyvoxFilamentApi.cpp +++ b/ios/src/PolyvoxFilamentApi.cpp @@ -362,7 +362,8 @@ extern "C" { int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshNames, + int numMeshTargets, float frameLengthInMs) { //std::packaged_task lambda([=]() mutable { ((AssetManager*)assetManager)->setBoneAnimationBuffer( @@ -371,7 +372,8 @@ extern "C" { numFrames, numBones, boneNames, - meshName, + meshNames, + numMeshTargets, frameLengthInMs ); //}); diff --git a/lib/animations/bone_animation_data.dart b/lib/animations/bone_animation_data.dart index 7bf9b7e3..672c7661 100644 --- a/lib/animations/bone_animation_data.dart +++ b/lib/animations/bone_animation_data.dart @@ -9,9 +9,9 @@ import 'package:vector_math/vector_math.dart'; /// class BoneAnimationData { final String boneName; - final String meshName; + final List meshNames; final Float32List frameData; double frameLengthInMs; BoneAnimationData( - this.boneName, this.meshName, this.frameData, this.frameLengthInMs); + this.boneName, this.meshNames, this.frameData, this.frameLengthInMs); } diff --git a/lib/animations/bone_driver.dart b/lib/animations/bone_driver.dart index e404efca..3ca30f5f 100644 --- a/lib/animations/bone_driver.dart +++ b/lib/animations/bone_driver.dart @@ -46,11 +46,21 @@ class BoneDriver { rotation.x *= weight; rotation.y *= weight; rotation.z *= weight; + rotation.w = 1; + return rotation; }).toList(); - yield rotations.fold( - rotations.first, (Quaternion a, Quaternion b) => a * b); + if (frameNum == 0) { + print(rotations); + } + + var result = rotations.fold( + rotations.first, (Quaternion a, Quaternion b) => a + b); + result.w = 1; + print("RESULT $result"); + yield result; + // .normalized(); // todo - bone translations } } diff --git a/lib/animations/csv_animation.dart b/lib/animations/csv_animation.dart index 1b9182e5..93b917bf 100644 --- a/lib/animations/csv_animation.dart +++ b/lib/animations/csv_animation.dart @@ -11,11 +11,12 @@ import 'package:vector_math/vector_math.dart'; /// A class for loading animation data from a single CSV and allocating between morph/bone animation with help. /// class DynamicAnimation { - final MorphAnimationData morphAnimation; + final MorphAnimationData? morphAnimation; final List boneAnimation; - factory DynamicAnimation.load(String meshName, String csvPath, + factory DynamicAnimation.load(String? meshName, String csvPath, {List? boneDrivers, + List? boneMeshes, String? boneDriverConfigPath, double? framerate}) { // create a MorphAnimationData instance from the given CSV @@ -23,8 +24,9 @@ class DynamicAnimation { var frameLengthInMs = 1000 / (framerate ?? 60.0); var morphNames = llf .item1; //.where((name) => !boneDrivers.any((element) => element.blendshape == name)); - var morphAnimationData = - MorphAnimationData(meshName, llf.item2, morphNames, frameLengthInMs); + + var morphAnimationData = MorphAnimationData( + meshName ?? "NULL", llf.item2, morphNames, frameLengthInMs); final boneAnimations = []; @@ -45,14 +47,14 @@ class DynamicAnimation { // iterate over every bone driver if (boneDrivers != null) { for (var driver in boneDrivers) { - // get all frames for the single the blendshape + // collect the frame data for the blendshapes that this driver uses var morphData = driver.transformations .map((String blendshape, Transformation transformation) { return MapEntry( blendshape, morphAnimationData.getData(blendshape).toList()); }); - // apply the driver to the blendshape weight + // apply the driver to the frame data var transformedQ = driver.transform(morphData).toList(); // transform the quaternion to a Float32List @@ -60,7 +62,7 @@ class DynamicAnimation { // add to the list of boneAnimations boneAnimations.add(BoneAnimationData( - driver.bone, meshName, transformedF, frameLengthInMs)); + driver.bone, boneMeshes!, transformedF, frameLengthInMs)); } } diff --git a/lib/animations/live_link_face_bone_driver.dart b/lib/animations/live_link_face_bone_driver.dart index b5cc9c99..921b8583 100644 --- a/lib/animations/live_link_face_bone_driver.dart +++ b/lib/animations/live_link_face_bone_driver.dart @@ -5,8 +5,9 @@ import 'package:vector_math/vector_math.dart'; BoneDriver getLiveLinkFaceBoneDrivers(String bone) { return BoneDriver(bone, { - "HeadPitch": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)), - "HeadRoll": Transformation(Quaternion.axisAngle(Vector3(0, 0, 1), pi / 2)), + "HeadPitch": + Transformation(Quaternion.axisAngle(Vector3(0, 0, -1), pi / 3)), + "HeadRoll": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)), "HeadYaw": Transformation(Quaternion.axisAngle(Vector3(0, 1, 0), pi / 2)), }); } diff --git a/lib/filament_controller.dart b/lib/filament_controller.dart index 52df302b..561862a9 100644 --- a/lib/filament_controller.dart +++ b/lib/filament_controller.dart @@ -320,32 +320,26 @@ class FilamentController { /// Animates morph target weights/bone transforms (where each frame requires a duration of [frameLengthInMs]. /// [morphWeights] is a list of doubles in frame-major format. /// Each frame is [numWeights] in length, and each entry is the weight to be applied to the morph target located at that index in the mesh primitive at that frame. + /// for now we only allow animating a single bone (though multiple skinned targets are supported) /// void setBoneAnimation( - FilamentEntity asset, List animations) async { - // for future compatibility, instances of BoneAnimationData can specify individual mesh targets - // however on the rendering side we currently only allow one set of frame data for one mesh target (though multiple bones are supported). - // this is a check that all animations are targeting the same mesh - assert(animations.map((e) => e.meshName).toSet().length == 1); - - var data = - calloc(animations.length * animations.first.frameData.length); + FilamentEntity asset, BoneAnimationData animation) async { + var data = calloc(animation.frameData.length); int offset = 0; - var numFrames = animations.first.frameData.length ~/ 7; - var boneNames = calloc>(animations.length); - int animIdx = 0; - for (var animation in animations) { - if (animation.frameData.length ~/ 7 != numFrames) { - throw Exception( - "All bone animations must share the same animation frame data length."); - } - for (int i = 0; i < animation.frameData.length; i++) { - data.elementAt(offset).value = animation.frameData[i]; - offset += 1; - } - boneNames.elementAt(animIdx).value = - animation.boneName.toNativeUtf8().cast(); - animIdx++; + var numFrames = animation.frameData.length ~/ 7; + var boneNames = calloc>(1); + boneNames.elementAt(0).value = + animation.boneName.toNativeUtf8().cast(); + + var meshNames = calloc>(animation.meshNames.length); + for (int i = 0; i < animation.meshNames.length; i++) { + meshNames.elementAt(i).value = + animation.meshNames[i].toNativeUtf8().cast(); + } + + for (int i = 0; i < animation.frameData.length; i++) { + data.elementAt(offset).value = animation.frameData[i]; + offset += 1; } _nativeLibrary.set_bone_animation( @@ -353,10 +347,11 @@ class FilamentController { asset, data, numFrames, - animations.length, + 1, boneNames, - animations.first.meshName.toNativeUtf8().cast(), - animations.first.frameLengthInMs); + meshNames, + animation.meshNames.length, + animation.frameLengthInMs); calloc.free(data); } diff --git a/lib/generated_bindings.dart b/lib/generated_bindings.dart index 2197eb07..1f807909 100644 --- a/lib/generated_bindings.dart +++ b/lib/generated_bindings.dart @@ -659,7 +659,8 @@ class NativeLibrary { int numFrames, int numBones, ffi.Pointer> boneNames, - ffi.Pointer meshName, + ffi.Pointer> meshName, + int numMeshTargets, double frameLengthInMs, ) { return _set_bone_animation( @@ -670,6 +671,7 @@ class NativeLibrary { numBones, boneNames, meshName, + numMeshTargets, frameLengthInMs, ); } @@ -683,7 +685,8 @@ class NativeLibrary { ffi.Int, ffi.Int, ffi.Pointer>, - ffi.Pointer, + ffi.Pointer>, + ffi.Int, ffi.Float)>>('set_bone_animation'); late final _set_bone_animation = _set_bone_animationPtr.asFunction< void Function( @@ -693,7 +696,8 @@ class NativeLibrary { int, int, ffi.Pointer>, - ffi.Pointer, + ffi.Pointer>, + int, double)>(); void play_animation(