From fae619feb89bd8aece6e56f1cf0aa9fd4e317b0d Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 26 Apr 2023 17:49:44 +0800 Subject: [PATCH] migrate some animation helper classes --- lib/animations/animation_builder.dart | 47 +++++++--- lib/animations/animations.dart | 61 ------------ lib/animations/bone_animation_data.dart | 16 ++++ lib/animations/bone_driver.dart | 57 ++++++++++++ lib/animations/csv_animation.dart | 113 +++++++++++++++++++++++ lib/animations/morph_animation_data.dart | 31 +++++++ test/bone_driver_test.dart | 73 +++++++++++++++ 7 files changed, 326 insertions(+), 72 deletions(-) delete mode 100644 lib/animations/animations.dart create mode 100644 lib/animations/bone_animation_data.dart create mode 100644 lib/animations/bone_driver.dart create mode 100644 lib/animations/csv_animation.dart create mode 100644 lib/animations/morph_animation_data.dart create mode 100644 test/bone_driver_test.dart diff --git a/lib/animations/animation_builder.dart b/lib/animations/animation_builder.dart index 4df2a40a..45455cac 100644 --- a/lib/animations/animation_builder.dart +++ b/lib/animations/animation_builder.dart @@ -1,13 +1,13 @@ -import 'package:polyvox_filament/animations/animations.dart'; +import 'package:polyvox_filament/animations/bone_animation_data.dart'; +import 'package:polyvox_filament/animations/morph_animation_data.dart'; import 'package:polyvox_filament/filament_controller.dart'; -import 'package:tuple/tuple.dart'; import 'package:flutter/foundation.dart'; import 'package:vector_math/vector_math.dart'; class AnimationBuilder { final FilamentController controller; - DartBoneAnimation? dartBoneAnimation; + BoneAnimationData? BoneAnimationData; double _frameLengthInMs = 0; double _duration = 0; @@ -16,7 +16,7 @@ class AnimationBuilder { double? _interpMorphStartValue; double? _interpMorphEndValue; - List? _dartBoneAnimations = null; + List? _BoneAnimationDatas = null; FilamentEntity asset; String meshName; @@ -53,11 +53,11 @@ class AnimationBuilder { } var morphAnimation = - MorphAnimation(meshName, morphData, morphNames, _frameLengthInMs); - print("SETTING!"); - controller.setMorphAnimation(asset, morphAnimation); - // return Tuple2>( - // morphAnimation, _dartBoneAnimations!); + MorphAnimationData(meshName, morphData, morphNames, _frameLengthInMs); + + controller.setMorphAnimationData(asset, morphAnimation); + // return Tuple2>( + // morphAnimation, _BoneAnimationDatas!); } AnimationBuilder setDuration(double secs) { @@ -118,15 +118,40 @@ class AnimationBuilder { // var boneFrameData = BoneTransformFrameData(translations, quats); - // _DartBoneAnimations ??= []; + // _BoneAnimationDatas ??= []; // var frameData = List>.generate( // numFrames, (index) => boneFrameData.getFrameData(index).toList()); // var animData = Float32List.fromList(frameData.expand((x) => x).toList()); - // _DartBoneAnimations!.add(DartDartBoneAnimation([boneName], [meshName], animData)); + // _BoneAnimationDatas!.add(DartBoneAnimationData([boneName], [meshName], animData)); return this; } } + +class BoneTransformFrameData { + final List translations; + final List quaternions; + + /// + /// The length of [translations] and [quaternions] must be the same; + /// each entry represents the Vec3/Quaternion for the given frame. + /// + BoneTransformFrameData(this.translations, this.quaternions) { + if (translations.length != quaternions.length) { + throw Exception("Length of translation/quaternion frames must match"); + } + } + + Iterable getFrameData(int frame) sync* { + yield translations[frame].x; + yield translations[frame].y; + yield translations[frame].z; + yield quaternions[frame].x; + yield quaternions[frame].y; + yield quaternions[frame].z; + yield quaternions[frame].w; + } +} diff --git a/lib/animations/animations.dart b/lib/animations/animations.dart deleted file mode 100644 index 41d1edd6..00000000 --- a/lib/animations/animations.dart +++ /dev/null @@ -1,61 +0,0 @@ -import 'dart:typed_data'; - -import 'package:vector_math/vector_math.dart'; - -class DartBoneAnimation { - final String boneName; - final String meshName; - final Float32List frameData; - double frameLengthInMs; - DartBoneAnimation( - this.boneName, this.meshName, this.frameData, this.frameLengthInMs); -} - -// -// Frame weights for the morph targets specified in [morphNames] attached to mesh [meshName]. -// morphData is laid out as numFrames x numMorphTargets -// where the weights are in the same order as [morphNames]. -// [morphNames] must be provided but is not used directly; this is only used to check that the eventual asset being animated contains the same morph targets in the same order. -// -class MorphAnimation { - final String meshName; - final List morphNames; - - final Float32List data; - - MorphAnimation( - this.meshName, this.data, this.morphNames, this.frameLengthInMs) { - assert(data.length == morphNames.length * numFrames); - } - - int get numMorphWeights => morphNames.length; - - int get numFrames => data.length ~/ numMorphWeights; - - final double frameLengthInMs; -} - -class BoneTransformFrameData { - final List translations; - final List quaternions; - - /// - /// The length of [translations] and [quaternions] must be the same; - /// each entry represents the Vec3/Quaternion for the given frame. - /// - BoneTransformFrameData(this.translations, this.quaternions) { - if (translations.length != quaternions.length) { - throw Exception("Length of translation/quaternion frames must match"); - } - } - - Iterable getFrameData(int frame) sync* { - yield translations[frame].x; - yield translations[frame].y; - yield translations[frame].z; - yield quaternions[frame].x; - yield quaternions[frame].y; - yield quaternions[frame].z; - yield quaternions[frame].w; - } -} diff --git a/lib/animations/bone_animation_data.dart b/lib/animations/bone_animation_data.dart new file mode 100644 index 00000000..cb694b01 --- /dev/null +++ b/lib/animations/bone_animation_data.dart @@ -0,0 +1,16 @@ +import 'dart:typed_data'; +import 'package:vector_math/vector_math.dart'; + +/// +/// Model class for bone animation frame data. +/// To create dynamic/runtime bone animations (as distinct from animations embedded in a glTF asset), create an instance containing the relevant +/// data and pass to the [setBoneAnimation] method on a [FilamentController]. +/// +class BoneAnimationData { + final String boneName; + final String meshName; + final Float32List frameData; + double frameLengthInMs; + BoneAnimationData( + this.boneName, this.meshName, this.frameData, this.frameLengthInMs); +} diff --git a/lib/animations/bone_driver.dart b/lib/animations/bone_driver.dart new file mode 100644 index 00000000..eeb0237a --- /dev/null +++ b/lib/animations/bone_driver.dart @@ -0,0 +1,57 @@ +import 'dart:convert'; +import 'package:vector_math/vector_math.dart'; +import 'package:flutter/foundation.dart'; +import 'package:vector_math/vector_math.dart'; + +/// +/// Some animation data may be specified as blendshape weights (say, between -1 and 1) +/// but at runtime we want to retarget this to drive a bone translation/rotation (say, between -pi/2 and pi/2). +/// A [BoneDriver] is our mechanism for translating the former to the latter, containing: +/// 1) a blendshape name +/// 2) a bone name +/// 3) min/max translation values (corresponding to -1/1 on the blendshape), and +/// 4) min/max rotation values (corresponding to -1/1 on the blendshape) +/// + +class BoneDriver { + final String bone; + final String blendshape; + + late final Vector3 transMin; + late final Vector3 transMax; + late final Quaternion rotMin; + late final Quaternion rotMax; + + BoneDriver(this.bone, this.blendshape, this.rotMin, this.rotMax, + Vector3? transMin, Vector3? transMax) { + this.transMin = transMin ?? Vector3.zero(); + this.transMax = transMax ?? Vector3.zero(); + } + + factory BoneDriver.fromJsonObject(dynamic jsonObject) { + return BoneDriver( + jsonObject["bone"], + jsonObject["blendshape"], + Quaternion.fromFloat32List(Float32List.fromList(jsonObject["rotMin"])), + Quaternion.fromFloat32List(Float32List.fromList(jsonObject["rotMax"])), + Vector3.fromFloat32List(Float32List.fromList(jsonObject["transMin"])), + Vector3.fromFloat32List(Float32List.fromList(jsonObject["transMax"])), + ); + } + + // + // Accepts a Float32List containing [numFrames] frames of data for a single morph target weight (for efficiency, this must be unravelled to a single contiguous Float32List). + // Returns a generator that yields [numFrames] Quaternions, each representing the (weighted) rotation/translation specified by the mapping of this BoneDriver. + // + Iterable transform(List morphTargetFrameData) sync* { + for (int i = 0; i < morphTargetFrameData.length; i++) { + var weight = (morphTargetFrameData[i] / 2) + 0.5; + + yield Quaternion( + rotMin.x + (weight * (rotMax.x - rotMin.x)), + rotMin.y + (weight * (rotMax.y - rotMin.y)), + rotMin.z + (weight * (rotMax.z - rotMin.z)), + 1.0); + } + } +} diff --git a/lib/animations/csv_animation.dart b/lib/animations/csv_animation.dart new file mode 100644 index 00000000..b46d35d5 --- /dev/null +++ b/lib/animations/csv_animation.dart @@ -0,0 +1,113 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; +import 'package:tuple/tuple.dart'; +import 'package:polyvox_filament/animations/bone_animation_data.dart'; +import 'package:polyvox_filament/animations/bone_driver.dart'; +import 'package:polyvox_filament/animations/morph_animation_data.dart'; +import 'package:vector_math/vector_math.dart'; + +/// +/// A class for loading animation data from a single CSV and allocating between morph/bone animation with help. +/// +class DynamicAnimation { + final MorphAnimationData morphAnimation; + final List boneAnimation; + + factory DynamicAnimation.load(String meshName, String csvPath, + {String? boneDriverConfigPath}) { + // create a MorphAnimationData instance from the given CSV + var llf = _loadLiveLinkFaceCSV(csvPath); + var morphNames = llf + .item1; //.where((name) => !boneDrivers.any((element) => element.blendshape == name)); + var morphAnimationData = MorphAnimationData( + meshName, + llf.item2, + morphNames, + 1000 / 60.0, + ); + + final boneAnimations = []; + + // if applicable, load the bone driver config + if (boneDriverConfigPath != null) { + var boneData = json.decode(File(boneDriverConfigPath).readAsStringSync()); + // for each driver + for (var key in boneData.keys()) { + var driver = BoneDriver.fromJsonObject(boneData[key]); + + // get all frames for the single the blendshape + var morphFrameData = + morphAnimationData.getData(driver.blendshape).toList(); + + // apply the driver to the blendshape weight + var transformedQ = driver.transform(morphFrameData).toList(); + + // transform the quaternion to a Float32List + var transformedF = _quaternionToFloatList(transformedQ); + + // add to the list of boneAnimations + boneAnimations.add(BoneAnimationData( + driver.bone, meshName, transformedF, 1000.0 / 60.0)); + } + } + + return DynamicAnimation(morphAnimationData, boneAnimations); + } + + static Float32List _quaternionToFloatList(List quats) { + var data = Float32List(quats.length * 4); + for (var quat in quats) { + data.addAll([0, 0, 0, quat.w, quat.x, quat.y, quat.z]); + } + return data; + } + + DynamicAnimation(this.morphAnimation, this.boneAnimation); + + /// + /// Load visemes fom a CSV file formatted according to the following header: + /// "Timecode,BlendShapeCount,EyeBlinkLeft,EyeLookDownLeft,EyeLookInLeft,EyeLookOutLeft,EyeLookUpLeft,EyeSquintLeft,EyeWideLeft,EyeBlinkRight,EyeLookDownRight,EyeLookInRight,EyeLookOutRight,EyeLookUpRight,EyeSquintRight,EyeWideRight,JawForward,JawRight,JawLeft,JawOpen,MouthClose,MouthFunnel,MouthPucker,MouthRight,MouthLeft,MouthSmileLeft,MouthSmileRight,MouthFrownLeft,MouthFrownRight,MouthDimpleLeft,MouthDimpleRight,MouthStretchLeft,MouthStretchRight,MouthRollLower,MouthRollUpper,MouthShrugLower,MouthShrugUpper,MouthPressLeft,MouthPressRight,MouthLowerDownLeft,MouthLowerDownRight,MouthUpperUpLeft,MouthUpperUpRight,BrowDownLeft,BrowDownRight,BrowInnerUp,BrowOuterUpLeft,BrowOuterUpRight,CheekPuff,CheekSquintLeft,CheekSquintRight,NoseSneerLeft,NoseSneerRight,TongueOut,HeadYaw,HeadPitch,HeadRoll,LeftEyeYaw,LeftEyePitch,LeftEyeRoll,RightEyeYaw,RightEyePitch,RightEyeRoll" + /// Returns only those specified by [targetNames]. + /// + static Tuple2, Float32List> _loadLiveLinkFaceCSV(String path) { + final data = File(path) + .readAsLinesSync() + .where((l) => l.length > 1) + .map((l) => l.split(",")); + + final header = data.first; + final numBlendShapes = header.length - 2; + + final _data = []; + + for (var frame in data.skip(1)) { + int numFrameWeights = frame.length - 2; + // CSVs may contain rows where the "BlendShapeCount" column is set to "0" and/or the weight columns are simply missing. + // This can happen when something went wrong while recording via an app (e.g. LiveLinkFace) + // Whenever we encounter this type of row, we consider that all weights should be set to zero for that frame. + if (numFrameWeights == int.parse(frame[1])) { + _data.addAll(List.filled(numBlendShapes, 0.0)); + continue; + } + + // + // Now, we check that the actual number of weight columns matches the header + // we ignore the "BlendShapeCount" column (and just count the number of columns) + // This is due to some legacy issues where we generated CSVs that had 61 weight columns, but accidentally left the "BlendShapeCount" column at 55 + // This is probably fine as we always have either zero weights (handled above), or all weights (handled below). + // In other words, if this throws, we have a serious problem. + if (numFrameWeights != numBlendShapes) { + throw Exception( + "Malformed CSV, header specifies ${numBlendShapes} columns but frame specified only $numFrameWeights weights"); + } + + _data.addAll(frame + .skip(2) + .map((weight) => double.parse(weight)) + .cast() + .toList()); + } + return Tuple2(header.skip(2).toList(), Float32List.fromList(_data)); + } +} diff --git a/lib/animations/morph_animation_data.dart b/lib/animations/morph_animation_data.dart new file mode 100644 index 00000000..ac6ab4a2 --- /dev/null +++ b/lib/animations/morph_animation_data.dart @@ -0,0 +1,31 @@ +// +// Frame weights for the morph targets specified in [morphNames] attached to mesh [meshName]. +// morphData is laid out as numFrames x numMorphTargets +// where the weights are in the same order as [morphNames]. +// [morphNames] must be provided but is not used directly; this is only used to check that the eventual asset being animated contains the same morph targets in the same order. +// +import 'dart:typed_data'; + +class MorphAnimationData { + final String meshName; + final List morphNames; + + final Float32List data; + + MorphAnimationData( + this.meshName, this.data, this.morphNames, this.frameLengthInMs) { + assert(data.length == morphNames.length * numFrames); + } + + int get numMorphWeights => morphNames.length; + + int get numFrames => data.length ~/ numMorphWeights; + + final double frameLengthInMs; + + Iterable getData(String morphName) sync* { + for (int i = 0; i < numFrames; i++) { + yield data[i * numMorphWeights]; + } + } +} diff --git a/test/bone_driver_test.dart b/test/bone_driver_test.dart new file mode 100644 index 00000000..09a90305 --- /dev/null +++ b/test/bone_driver_test.dart @@ -0,0 +1,73 @@ +import 'dart:convert'; +import 'dart:math'; +import 'dart:typed_data'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:polyvox_filament/animations/bone_driver.dart'; +import 'package:vector_math/vector_math.dart'; + +void main() { + group('BoneDriver', () { + test('constructor sets correct values', () { + Quaternion rotMin = Quaternion.identity(); + Quaternion rotMax = Quaternion.axisAngle(Vector3(1, 0, 0), 0.5); + Vector3 transMin = Vector3.zero(); + Vector3 transMax = Vector3(1, 1, 1); + + BoneDriver boneDriver = BoneDriver( + 'bone1', 'blendshape1', rotMin, rotMax, transMin, transMax); + + expect(boneDriver.bone, 'bone1'); + expect(boneDriver.blendshape, 'blendshape1'); + expect(boneDriver.rotMin, rotMin); + expect(boneDriver.rotMax, rotMax); + expect(boneDriver.transMin, transMin); + expect(boneDriver.transMax, transMax); + }); + + test('fromJsonObject creates BoneDriver instance correctly', () { + dynamic jsonObject = { + "bone": "bone1", + "blendshape": "blendshape1", + "rotMin": Quaternion.identity().storage, + "rotMax": Quaternion.axisAngle(Vector3(1, 0, 0), 0.5).storage, + "transMin": Vector3.zero().storage, + "transMax": Vector3(1, 1, 1).storage + }; + + BoneDriver boneDriver = BoneDriver.fromJsonObject(jsonObject); + + expect(boneDriver.bone, 'bone1'); + expect(boneDriver.blendshape, 'blendshape1'); + expect(boneDriver.rotMin.absoluteError(Quaternion.identity()), 0); + expect( + boneDriver.rotMax + .absoluteError(Quaternion.axisAngle(Vector3(1, 0, 0), 0.5)), + 0); + expect(boneDriver.transMin.absoluteError(Vector3.zero()), 0); + expect(boneDriver.transMax.absoluteError(Vector3(1, 1, 1)), 0); + }); + + test('transform generates correct Quaternions', () { + Quaternion rotMin = Quaternion.identity(); + Quaternion rotMax = Quaternion.axisAngle(Vector3(1, 0, 0), 0.5); + BoneDriver boneDriver = + BoneDriver('bone1', 'blendshape1', rotMin, rotMax, null, null); + + List morphTargetFrameData = [-1, 0, 1]; + List expectedResult = [ + Quaternion(rotMin.x, rotMin.y, rotMin.z, 1.0), + Quaternion((rotMin.x + rotMax.x) / 2, (rotMin.y + rotMax.y) / 2, + (rotMin.z + rotMax.z) / 2, 1.0), + Quaternion(rotMax.x, rotMax.y, rotMax.z, 1.0), + ]; + + Iterable result = boneDriver.transform(morphTargetFrameData); + List resultAsList = result.toList(); + expect(resultAsList.length, expectedResult.length); + + for (int i = 0; i < expectedResult.length; i++) { + expect(resultAsList[i].absoluteError(expectedResult[i]), 0); + } + }); + }); +}