From 50c47fe908b093b4856128fa2dc465f8b2f6e018 Mon Sep 17 00:00:00 2001 From: Nick Fisher Date: Wed, 18 Oct 2023 14:37:45 +0800 Subject: [PATCH] move LiveLinkFace-related data loaders to viewer project --- example/lib/main.dart | 3 +- lib/animations/animation_builder.dart | 5 +- ...nimation_data.dart => animation_data.dart} | 17 +++ lib/animations/bone_animation_data.dart | 17 --- lib/animations/bone_driver.dart | 79 ---------- lib/animations/csv_animation.dart | 141 ------------------ .../live_link_face_bone_driver.dart | 13 -- lib/filament_controller.dart | 4 +- lib/filament_controller_ffi.dart | 4 +- lib/filament_controller_method_channel.dart | 4 +- test/bone_driver_test.dart | 70 --------- 11 files changed, 26 insertions(+), 331 deletions(-) rename lib/animations/{morph_animation_data.dart => animation_data.dart} (67%) delete mode 100644 lib/animations/bone_animation_data.dart delete mode 100644 lib/animations/bone_driver.dart delete mode 100644 lib/animations/csv_animation.dart delete mode 100644 lib/animations/live_link_face_bone_driver.dart delete mode 100644 test/bone_driver_test.dart diff --git a/example/lib/main.dart b/example/lib/main.dart index 12f60d83..b464d3b9 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -3,9 +3,10 @@ import 'dart:io'; import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:permission_handler/permission_handler.dart'; +import 'package:polyvox_filament/animations/animation_data.dart'; import 'package:polyvox_filament/filament_controller.dart'; -import 'package:polyvox_filament/animations/bone_animation_data.dart'; + import 'package:polyvox_filament/filament_controller_ffi.dart'; import 'package:polyvox_filament/animations/animation_builder.dart'; diff --git a/lib/animations/animation_builder.dart b/lib/animations/animation_builder.dart index 8c396e11..da48b6ce 100644 --- a/lib/animations/animation_builder.dart +++ b/lib/animations/animation_builder.dart @@ -1,7 +1,4 @@ -import 'package:polyvox_filament/animations/morph_animation_data.dart'; -import 'package:polyvox_filament/filament_controller_method_channel.dart'; - -import 'package:flutter/foundation.dart'; +import 'package:polyvox_filament/animations/animation_data.dart'; import 'package:vector_math/vector_math.dart'; class AnimationBuilder { diff --git a/lib/animations/morph_animation_data.dart b/lib/animations/animation_data.dart similarity index 67% rename from lib/animations/morph_animation_data.dart rename to lib/animations/animation_data.dart index a7df9516..90e8cf7e 100644 --- a/lib/animations/morph_animation_data.dart +++ b/lib/animations/animation_data.dart @@ -5,6 +5,8 @@ // the morph targets specified in [morphNames] attached to mesh [meshName]. // [animatedMorphNames] must be provided but is not used directly; this is only used to check that the eventual asset being animated contains the same morph targets in the same order. // +import 'dart:typed_data'; + class MorphAnimationData { final String meshName; final List animatedMorphNames; @@ -30,3 +32,18 @@ class MorphAnimationData { } } } + +/// +/// Model class for bone animation frame data. +/// To create dynamic/runtime bone animations (as distinct from animations embedded in a glTF asset), create an instance containing the relevant +/// data and pass to the [setBoneAnimation] method on a [FilamentController]. +/// [frameData] is laid out as [locX, locY, locZ, rotW, rotX, rotY, rotZ] +/// +class BoneAnimationData { + final String boneName; + final List meshNames; + final Float32List frameData; + double frameLengthInMs; + BoneAnimationData( + this.boneName, this.meshNames, this.frameData, this.frameLengthInMs); +} diff --git a/lib/animations/bone_animation_data.dart b/lib/animations/bone_animation_data.dart deleted file mode 100644 index 672c7661..00000000 --- a/lib/animations/bone_animation_data.dart +++ /dev/null @@ -1,17 +0,0 @@ -import 'dart:typed_data'; -import 'package:vector_math/vector_math.dart'; - -/// -/// Model class for bone animation frame data. -/// To create dynamic/runtime bone animations (as distinct from animations embedded in a glTF asset), create an instance containing the relevant -/// data and pass to the [setBoneAnimation] method on a [FilamentController]. -/// [frameData] is laid out as [locX, locY, locZ, rotW, rotX, rotY, rotZ] -/// -class BoneAnimationData { - final String boneName; - final List meshNames; - final Float32List frameData; - double frameLengthInMs; - BoneAnimationData( - this.boneName, this.meshNames, this.frameData, this.frameLengthInMs); -} diff --git a/lib/animations/bone_driver.dart b/lib/animations/bone_driver.dart deleted file mode 100644 index 7a78abf3..00000000 --- a/lib/animations/bone_driver.dart +++ /dev/null @@ -1,79 +0,0 @@ -import 'dart:convert'; -import 'package:vector_math/vector_math.dart'; -import 'package:flutter/foundation.dart'; -import 'package:vector_math/vector_math.dart'; - -/// -/// Some animation data may be specified as blendshape weights (say, between -1 and 1) -/// but at runtime we want to retarget this to drive a bone translation/rotation (say, between -pi/2 and pi/2). -/// A [BoneDriver] is our mechanism for translating the former to the latter, containing: -/// 1) a blendshape name -/// 2) a bone name -/// 3) min/max translation values (corresponding to -1/1 on the blendshape), and -/// 4) min/max rotation values (corresponding to -1/1 on the blendshape) -/// - -class Transformation { - final Quaternion rotation; - late final Vector3 translation; - - Transformation(this.rotation, {Vector3? translation}) { - this.translation = translation ?? Vector3.zero(); - } -} - -class BoneDriver { - final String bone; - final Map - transformations; // maps a blendshape key to a Transformation - - BoneDriver(this.bone, this.transformations); - - // - // Accepts a Float32List containing [numFrames] frames of data for a single morph target weight (for efficiency, this must be unravelled to a single contiguous Float32List). - // Returns a generator that yields [numFrames] Quaternions, each representing the (weighted) rotation/translation specified by the mapping of this BoneDriver. - // - Iterable transform( - Map> morphTargetFrameData) sync* { - assert(setEquals( - morphTargetFrameData.keys.toSet(), transformations.keys.toSet())); - var numFrames = morphTargetFrameData.values.first.length; - assert(morphTargetFrameData.values.every((x) => x.length == numFrames)); - for (int frameNum = 0; frameNum < numFrames; frameNum++) { - var rotations = transformations.keys.map((blendshape) { - var weight = morphTargetFrameData[blendshape]![frameNum]; - var rotation = transformations[blendshape]!.rotation.clone(); - rotation.x *= weight; - rotation.y *= weight; - rotation.z *= weight; - rotation.w = 1; - - return rotation; - }).toList(); - - var result = rotations.fold( - rotations.first, (Quaternion a, Quaternion b) => a + b); - result.w = 1; - yield result; - } - } - - factory BoneDriver.fromJsonObject(dynamic jsonObject) { - throw Exception("TODO"); - // return BoneDriver( - // jsonObject["bone"], - // Map.fromIterable(jsonObject["blendshape"].map((bsName, quats) { - // var q = quats.map(()) - // MapEntry(k, - } -} - - - - - // } - // yield Quaternion( - // rotMin.x + (weight * (rotMax.x - rotMin.x)), - // rotMin.y + (weight * (rotMax.y - rotMin.y)), - // rotMin.z + (weight * (rotMax.z - rotMin.z)), - // 1.0); \ No newline at end of file diff --git a/lib/animations/csv_animation.dart b/lib/animations/csv_animation.dart deleted file mode 100644 index 09374ed2..00000000 --- a/lib/animations/csv_animation.dart +++ /dev/null @@ -1,141 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; -import 'package:tuple/tuple.dart'; -import 'package:polyvox_filament/animations/bone_animation_data.dart'; -import 'package:polyvox_filament/animations/bone_driver.dart'; -import 'package:polyvox_filament/animations/morph_animation_data.dart'; -import 'package:vector_math/vector_math.dart'; - -/// -/// A class for loading animation data from a single CSV and allocating between morph/bone animation. -/// -class DynamicAnimation { - final MorphAnimationData? morphAnimation; - final List boneAnimation; - - factory DynamicAnimation.load(String? meshName, String csvPath, - {List? boneDrivers, - List? boneMeshes, - String? boneDriverConfigPath, - double? framerate}) { - // create a MorphAnimationData instance from the given CSV - var llf = _loadLiveLinkFaceCSV(csvPath); - var frameLengthInMs = 1000 / (framerate ?? 60.0); - var morphNames = llf.item1; - - if (boneDrivers != null) { - morphNames = morphNames - .where((name) => - boneDrivers!.any((element) => element.bone == name) == false) - .toList(); - } - - var morphAnimationData = MorphAnimationData( - meshName ?? "NULL", - llf.item2, - morphNames, - List.generate(morphNames.length, (index) => index), - frameLengthInMs); - - final boneAnimations = []; - - // if applicable, load the bone driver config - if (boneDriverConfigPath != null) { - if (boneDrivers != null) { - throw Exception( - "Specify either boneDrivers, or the config path, not both"); - } - boneDrivers = [ - json - .decode(File(boneDriverConfigPath).readAsStringSync()) - .map(BoneDriver.fromJsonObject) - .toList() - ]; - } - - // iterate over every bone driver - if (boneDrivers != null) { - for (var driver in boneDrivers) { - // collect the frame data for the blendshapes that this driver uses - var morphData = driver.transformations - .map((String blendshape, Transformation transformation) { - return MapEntry( - blendshape, morphAnimationData.getData(blendshape).toList()); - }); - - // apply the driver to the frame data - var transformedQ = driver.transform(morphData).toList(); - - // transform the quaternion to a Float32List - var transformedF = _quaternionToFloatList(transformedQ); - - // add to the list of boneAnimations - boneAnimations.add(BoneAnimationData( - driver.bone, boneMeshes!, transformedF, frameLengthInMs)); - } - } - - return DynamicAnimation(morphAnimationData, boneAnimations); - } - - static Float32List _quaternionToFloatList(List quats) { - var data = Float32List(quats.length * 7); - int i = 0; - for (var quat in quats) { - data.setRange(i, i + 7, [0, 0, 0, quat.w, quat.x, quat.y, quat.z]); - i += 7; - } - return data; - } - - DynamicAnimation(this.morphAnimation, this.boneAnimation); - - /// - /// Load visemes fom a CSV file formatted according to the following header: - /// "Timecode,BlendShapeCount,EyeBlinkLeft,EyeLookDownLeft,EyeLookInLeft,EyeLookOutLeft,EyeLookUpLeft,EyeSquintLeft,EyeWideLeft,EyeBlinkRight,EyeLookDownRight,EyeLookInRight,EyeLookOutRight,EyeLookUpRight,EyeSquintRight,EyeWideRight,JawForward,JawRight,JawLeft,JawOpen,MouthClose,MouthFunnel,MouthPucker,MouthRight,MouthLeft,MouthSmileLeft,MouthSmileRight,MouthFrownLeft,MouthFrownRight,MouthDimpleLeft,MouthDimpleRight,MouthStretchLeft,MouthStretchRight,MouthRollLower,MouthRollUpper,MouthShrugLower,MouthShrugUpper,MouthPressLeft,MouthPressRight,MouthLowerDownLeft,MouthLowerDownRight,MouthUpperUpLeft,MouthUpperUpRight,BrowDownLeft,BrowDownRight,BrowInnerUp,BrowOuterUpLeft,BrowOuterUpRight,CheekPuff,CheekSquintLeft,CheekSquintRight,NoseSneerLeft,NoseSneerRight,TongueOut,HeadYaw,HeadPitch,HeadRoll,LeftEyeYaw,LeftEyePitch,LeftEyeRoll,RightEyeYaw,RightEyePitch,RightEyeRoll" - /// Returns two elements: - /// - a list containing the names of each blendshape/morph key - /// - a Float32List of length TxN, where T is the number of frames and N is the number of morph keys (i.e. the length of the list in the first element of the returned tuple). - /// - static Tuple2, Float32List> _loadLiveLinkFaceCSV(String path) { - final data = File(path) - .readAsLinesSync() - .where((l) => l.length > 1) - .map((l) => l.split(",")); - - final header = data.first; - final numBlendShapes = header.length - 2; - - final _data = []; - - for (var frame in data.skip(1)) { - int numFrameWeights = frame.length - 2; - // CSVs may contain rows where the "BlendShapeCount" column is set to "0" and/or the weight columns are simply missing. - // This can happen when something went wrong while recording via an app (e.g. LiveLinkFace) - // Whenever we encounter this type of row, we consider that all weights should be set to zero for that frame. - if (numFrameWeights != int.parse(frame[1])) { - _data.addAll(List.filled(numBlendShapes, 0.0)); - continue; - } - - // - // Now, we check that the actual number of weight columns matches the header - // we ignore the "BlendShapeCount" column (and just count the number of columns) - // This is due to some legacy issues where we generated CSVs that had 61 weight columns, but accidentally left the "BlendShapeCount" column at 55 - // This is probably fine as we always have either zero weights (handled above), or all weights (handled below). - // In other words, if this throws, we have a serious problem. - if (numFrameWeights != numBlendShapes) { - throw Exception( - "Malformed CSV, header specifies ${numBlendShapes} columns but frame specified only $numFrameWeights weights"); - } - - _data.addAll(frame - .skip(2) - .map((weight) => double.parse(weight)) - .cast() - .toList()); - } - return Tuple2(header.skip(2).toList(), Float32List.fromList(_data)); - } -} diff --git a/lib/animations/live_link_face_bone_driver.dart b/lib/animations/live_link_face_bone_driver.dart deleted file mode 100644 index 921b8583..00000000 --- a/lib/animations/live_link_face_bone_driver.dart +++ /dev/null @@ -1,13 +0,0 @@ -import 'dart:math'; - -import 'package:polyvox_filament/animations/bone_driver.dart'; -import 'package:vector_math/vector_math.dart'; - -BoneDriver getLiveLinkFaceBoneDrivers(String bone) { - return BoneDriver(bone, { - "HeadPitch": - Transformation(Quaternion.axisAngle(Vector3(0, 0, -1), pi / 3)), - "HeadRoll": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)), - "HeadYaw": Transformation(Quaternion.axisAngle(Vector3(0, 1, 0), pi / 2)), - }); -} diff --git a/lib/filament_controller.dart b/lib/filament_controller.dart index 9ff4f553..ddf51517 100644 --- a/lib/filament_controller.dart +++ b/lib/filament_controller.dart @@ -1,8 +1,8 @@ import 'dart:async'; import 'dart:ui' as ui; import 'package:flutter/services.dart'; -import 'package:polyvox_filament/animations/bone_animation_data.dart'; -import 'package:polyvox_filament/animations/morph_animation_data.dart'; + +import 'package:polyvox_filament/animations/animation_data.dart'; typedef FilamentEntity = int; diff --git a/lib/filament_controller_ffi.dart b/lib/filament_controller_ffi.dart index 1faa6be3..a4b747af 100644 --- a/lib/filament_controller_ffi.dart +++ b/lib/filament_controller_ffi.dart @@ -5,8 +5,8 @@ import 'dart:ui' as ui; import 'package:flutter/services.dart'; import 'package:ffi/ffi.dart'; import 'package:polyvox_filament/filament_controller.dart'; -import 'package:polyvox_filament/animations/bone_animation_data.dart'; -import 'package:polyvox_filament/animations/morph_animation_data.dart'; + +import 'package:polyvox_filament/animations/animation_data.dart'; import 'package:polyvox_filament/generated_bindings.dart'; const FilamentEntity _FILAMENT_ASSET_ERROR = 0; diff --git a/lib/filament_controller_method_channel.dart b/lib/filament_controller_method_channel.dart index 993f2212..f8b358af 100644 --- a/lib/filament_controller_method_channel.dart +++ b/lib/filament_controller_method_channel.dart @@ -2,8 +2,8 @@ import 'dart:async'; import 'dart:ui' as ui; import 'package:flutter/services.dart'; import 'package:polyvox_filament/filament_controller.dart'; -import 'package:polyvox_filament/animations/bone_animation_data.dart'; -import 'package:polyvox_filament/animations/morph_animation_data.dart'; + +import 'package:polyvox_filament/animations/animation_data.dart'; import 'package:polyvox_filament/generated_bindings_web.dart'; import 'filament_controller.dart'; diff --git a/test/bone_driver_test.dart b/test/bone_driver_test.dart deleted file mode 100644 index 1119876f..00000000 --- a/test/bone_driver_test.dart +++ /dev/null @@ -1,70 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; -import 'package:polyvox_filament/animations/bone_driver.dart'; -import 'package:vector_math/vector_math.dart'; - -void main() { - group('BoneDriver', () { - test( - 'transform should yield correct Quaternions for given morphTargetFrameData', - () { - final bone = 'bone1'; - final transformations = { - 'blendshape1': Transformation(Quaternion(1, 0, 0, 1)), - 'blendshape2': Transformation(Quaternion(0, 1, 0, 1)), - }; - final morphTargetFrameData = >{ - 'blendshape1': [0.5, -0.5], - 'blendshape2': [-1, 1], - }; - final boneDriver = BoneDriver(bone, transformations); - - final result = boneDriver.transform(morphTargetFrameData).toList(); - - expect(result.length, 2); - expect(result[0].x, -0.5); - expect(result[0].y, 0); - expect(result[0].z, -0.5); - expect(result[0].w, 0); - expect(result[1].x, 0.5); - expect(result[1].y, 0); - expect(result[1].z, 0.5); - expect(result[1].w, 0); - }); - - test( - 'transform should throw AssertionError when morphTargetFrameData keys do not match transformations keys', - () { - final bone = 'bone1'; - final transformations = { - 'blendshape1': Transformation(Quaternion(1, 0, 0, 0)), - 'blendshape2': Transformation(Quaternion(0, 1, 0, 0)), - }; - final morphTargetFrameData = >{ - 'blendshape1': [0.5, -0.5], - 'blendshape3': [-1, 1], - }; - final boneDriver = BoneDriver(bone, transformations); - - expect(() => boneDriver.transform(morphTargetFrameData), - throwsA(isA())); - }); - - test( - 'transform should throw AssertionError when morphTargetFrameData values lengths do not match', - () { - final bone = 'bone1'; - final transformations = { - 'blendshape1': Transformation(Quaternion(1, 0, 0, 0)), - 'blendshape2': Transformation(Quaternion(0, 1, 0, 0)), - }; - final morphTargetFrameData = >{ - 'blendshape1': [0.5, -0.5], - 'blendshape2': [-1], - }; - final boneDriver = BoneDriver(bone, transformations); - - expect(() => boneDriver.transform(morphTargetFrameData), - throwsA(isA())); - }); - }); -}