maobo/cup_edit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move LiveLinkFace-related data loaders to viewer project

Browse Source
This commit is contained in:
Nick Fisher
2023-10-18 14:37:45 +08:00
parent 721726d2d5
commit 50c47fe908
11 changed files with 26 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
example/lib/main.dart
View File

@@ -3,9 +3,10 @@ import 'dart:io';
import 'dart:typed_data'; import 'dart:typed_data';
import 'package:flutter/material.dart'; import 'package:flutter/material.dart';
import 'package:permission_handler/permission_handler.dart'; import 'package:permission_handler/permission_handler.dart';
import 'package:polyvox_filament/animations/animation_data.dart';
import 'package:polyvox_filament/filament_controller.dart'; import 'package:polyvox_filament/filament_controller.dart';
import 'package:polyvox_filament/animations/bone_animation_data.dart';
import 'package:polyvox_filament/filament_controller_ffi.dart'; import 'package:polyvox_filament/filament_controller_ffi.dart';
import 'package:polyvox_filament/animations/animation_builder.dart'; import 'package:polyvox_filament/animations/animation_builder.dart';

5
lib/animations/animation_builder.dart
View File

@@ -1,7 +1,4 @@
import 'package:polyvox_filament/animations/morph_animation_data.dart'; import 'package:polyvox_filament/animations/animation_data.dart';
import 'package:polyvox_filament/filament_controller_method_channel.dart';
import 'package:flutter/foundation.dart';
import 'package:vector_math/vector_math.dart'; import 'package:vector_math/vector_math.dart';
class AnimationBuilder { class AnimationBuilder {

17
lib/animations/morph_animation_data.dart → lib/animations/animation_data.dart
View File

@@ -5,6 +5,8 @@
// the morph targets specified in [morphNames] attached to mesh [meshName]. // the morph targets specified in [morphNames] attached to mesh [meshName].
// [animatedMorphNames] must be provided but is not used directly; this is only used to check that the eventual asset being animated contains the same morph targets in the same order. // [animatedMorphNames] must be provided but is not used directly; this is only used to check that the eventual asset being animated contains the same morph targets in the same order.
// //
import 'dart:typed_data';
class MorphAnimationData { class MorphAnimationData {
final String meshName; final String meshName;
final List<String> animatedMorphNames; final List<String> animatedMorphNames;
@@ -30,3 +32,18 @@ class MorphAnimationData {
} }
} }
} }
///
/// Model class for bone animation frame data.
/// To create dynamic/runtime bone animations (as distinct from animations embedded in a glTF asset), create an instance containing the relevant
/// data and pass to the [setBoneAnimation] method on a [FilamentController].
/// [frameData] is laid out as [locX, locY, locZ, rotW, rotX, rotY, rotZ]
///
class BoneAnimationData {
final String boneName;
final List<String> meshNames;
final Float32List frameData;
double frameLengthInMs;
BoneAnimationData(
this.boneName, this.meshNames, this.frameData, this.frameLengthInMs);
}

17
lib/animations/bone_animation_data.dart
View File

@@ -1,17 +0,0 @@
import 'dart:typed_data';
import 'package:vector_math/vector_math.dart';
///
/// Model class for bone animation frame data.
/// To create dynamic/runtime bone animations (as distinct from animations embedded in a glTF asset), create an instance containing the relevant
/// data and pass to the [setBoneAnimation] method on a [FilamentController].
/// [frameData] is laid out as [locX, locY, locZ, rotW, rotX, rotY, rotZ]
///
class BoneAnimationData {
final String boneName;
final List<String> meshNames;
final Float32List frameData;
double frameLengthInMs;
BoneAnimationData(
this.boneName, this.meshNames, this.frameData, this.frameLengthInMs);
}

79
lib/animations/bone_driver.dart
View File

@@ -1,79 +0,0 @@
import 'dart:convert';
import 'package:vector_math/vector_math.dart';
import 'package:flutter/foundation.dart';
import 'package:vector_math/vector_math.dart';
///
/// Some animation data may be specified as blendshape weights (say, between -1 and 1)
/// but at runtime we want to retarget this to drive a bone translation/rotation (say, between -pi/2 and pi/2).
/// A [BoneDriver] is our mechanism for translating the former to the latter, containing:
/// 1) a blendshape name
/// 2) a bone name
/// 3) min/max translation values (corresponding to -1/1 on the blendshape), and
/// 4) min/max rotation values (corresponding to -1/1 on the blendshape)
///
class Transformation {
final Quaternion rotation;
late final Vector3 translation;
Transformation(this.rotation, {Vector3? translation}) {
this.translation = translation ?? Vector3.zero();
}
}
class BoneDriver {
final String bone;
final Map<String, Transformation>
transformations; // maps a blendshape key to a Transformation
BoneDriver(this.bone, this.transformations);
//
// Accepts a Float32List containing [numFrames] frames of data for a single morph target weight (for efficiency, this must be unravelled to a single contiguous Float32List).
// Returns a generator that yields [numFrames] Quaternions, each representing the (weighted) rotation/translation specified by the mapping of this BoneDriver.
//
Iterable<Quaternion> transform(
Map<String, List<double>> morphTargetFrameData) sync* {
assert(setEquals(
morphTargetFrameData.keys.toSet(), transformations.keys.toSet()));
var numFrames = morphTargetFrameData.values.first.length;
assert(morphTargetFrameData.values.every((x) => x.length == numFrames));
for (int frameNum = 0; frameNum < numFrames; frameNum++) {
var rotations = transformations.keys.map((blendshape) {
var weight = morphTargetFrameData[blendshape]![frameNum];
var rotation = transformations[blendshape]!.rotation.clone();
rotation.x *= weight;
rotation.y *= weight;
rotation.z *= weight;
rotation.w = 1;
return rotation;
}).toList();
var result = rotations.fold(
rotations.first, (Quaternion a, Quaternion b) => a + b);
result.w = 1;
yield result;
}
}
factory BoneDriver.fromJsonObject(dynamic jsonObject) {
throw Exception("TODO");
// return BoneDriver(
// jsonObject["bone"],
// Map<String,Transformation>.fromIterable(jsonObject["blendshape"].map((bsName, quats) {
// var q = quats.map(())
// MapEntry(k,
}
}
// }
// yield Quaternion(
// rotMin.x + (weight * (rotMax.x - rotMin.x)),
// rotMin.y + (weight * (rotMax.y - rotMin.y)),
// rotMin.z + (weight * (rotMax.z - rotMin.z)),
// 1.0);

141
lib/animations/csv_animation.dart
View File

@@ -1,141 +0,0 @@
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:tuple/tuple.dart';
import 'package:polyvox_filament/animations/bone_animation_data.dart';
import 'package:polyvox_filament/animations/bone_driver.dart';
import 'package:polyvox_filament/animations/morph_animation_data.dart';
import 'package:vector_math/vector_math.dart';
///
/// A class for loading animation data from a single CSV and allocating between morph/bone animation.
///
class DynamicAnimation {
final MorphAnimationData? morphAnimation;
final List<BoneAnimationData> boneAnimation;
factory DynamicAnimation.load(String? meshName, String csvPath,
{List<BoneDriver>? boneDrivers,
List<String>? boneMeshes,
String? boneDriverConfigPath,
double? framerate}) {
// create a MorphAnimationData instance from the given CSV
var llf = _loadLiveLinkFaceCSV(csvPath);
var frameLengthInMs = 1000 / (framerate ?? 60.0);
var morphNames = llf.item1;
if (boneDrivers != null) {
morphNames = morphNames
.where((name) =>
boneDrivers!.any((element) => element.bone == name) == false)
.toList();
}
var morphAnimationData = MorphAnimationData(
meshName ?? "NULL",
llf.item2,
morphNames,
List<int>.generate(morphNames.length, (index) => index),
frameLengthInMs);
final boneAnimations = <BoneAnimationData>[];
// if applicable, load the bone driver config
if (boneDriverConfigPath != null) {
if (boneDrivers != null) {
throw Exception(
"Specify either boneDrivers, or the config path, not both");
}
boneDrivers = [
json
.decode(File(boneDriverConfigPath).readAsStringSync())
.map(BoneDriver.fromJsonObject)
.toList()
];
}
// iterate over every bone driver
if (boneDrivers != null) {
for (var driver in boneDrivers) {
// collect the frame data for the blendshapes that this driver uses
var morphData = driver.transformations
.map((String blendshape, Transformation transformation) {
return MapEntry(
blendshape, morphAnimationData.getData(blendshape).toList());
});
// apply the driver to the frame data
var transformedQ = driver.transform(morphData).toList();
// transform the quaternion to a Float32List
var transformedF = _quaternionToFloatList(transformedQ);
// add to the list of boneAnimations
boneAnimations.add(BoneAnimationData(
driver.bone, boneMeshes!, transformedF, frameLengthInMs));
}
}
return DynamicAnimation(morphAnimationData, boneAnimations);
}
static Float32List _quaternionToFloatList(List<Quaternion> quats) {
var data = Float32List(quats.length * 7);
int i = 0;
for (var quat in quats) {
data.setRange(i, i + 7, [0, 0, 0, quat.w, quat.x, quat.y, quat.z]);
i += 7;
}
return data;
}
DynamicAnimation(this.morphAnimation, this.boneAnimation);
///
/// Load visemes fom a CSV file formatted according to the following header:
/// "Timecode,BlendShapeCount,EyeBlinkLeft,EyeLookDownLeft,EyeLookInLeft,EyeLookOutLeft,EyeLookUpLeft,EyeSquintLeft,EyeWideLeft,EyeBlinkRight,EyeLookDownRight,EyeLookInRight,EyeLookOutRight,EyeLookUpRight,EyeSquintRight,EyeWideRight,JawForward,JawRight,JawLeft,JawOpen,MouthClose,MouthFunnel,MouthPucker,MouthRight,MouthLeft,MouthSmileLeft,MouthSmileRight,MouthFrownLeft,MouthFrownRight,MouthDimpleLeft,MouthDimpleRight,MouthStretchLeft,MouthStretchRight,MouthRollLower,MouthRollUpper,MouthShrugLower,MouthShrugUpper,MouthPressLeft,MouthPressRight,MouthLowerDownLeft,MouthLowerDownRight,MouthUpperUpLeft,MouthUpperUpRight,BrowDownLeft,BrowDownRight,BrowInnerUp,BrowOuterUpLeft,BrowOuterUpRight,CheekPuff,CheekSquintLeft,CheekSquintRight,NoseSneerLeft,NoseSneerRight,TongueOut,HeadYaw,HeadPitch,HeadRoll,LeftEyeYaw,LeftEyePitch,LeftEyeRoll,RightEyeYaw,RightEyePitch,RightEyeRoll"
/// Returns two elements:
/// - a list containing the names of each blendshape/morph key
/// - a Float32List of length TxN, where T is the number of frames and N is the number of morph keys (i.e. the length of the list in the first element of the returned tuple).
///
static Tuple2<List<String>, Float32List> _loadLiveLinkFaceCSV(String path) {
final data = File(path)
.readAsLinesSync()
.where((l) => l.length > 1)
.map((l) => l.split(","));
final header = data.first;
final numBlendShapes = header.length - 2;
final _data = <double>[];
for (var frame in data.skip(1)) {
int numFrameWeights = frame.length - 2;
// CSVs may contain rows where the "BlendShapeCount" column is set to "0" and/or the weight columns are simply missing.
// This can happen when something went wrong while recording via an app (e.g. LiveLinkFace)
// Whenever we encounter this type of row, we consider that all weights should be set to zero for that frame.
if (numFrameWeights != int.parse(frame[1])) {
_data.addAll(List<double>.filled(numBlendShapes, 0.0));
continue;
}
//
// Now, we check that the actual number of weight columns matches the header
// we ignore the "BlendShapeCount" column (and just count the number of columns)
// This is due to some legacy issues where we generated CSVs that had 61 weight columns, but accidentally left the "BlendShapeCount" column at 55
// This is probably fine as we always have either zero weights (handled above), or all weights (handled below).
// In other words, if this throws, we have a serious problem.
if (numFrameWeights != numBlendShapes) {
throw Exception(
"Malformed CSV, header specifies ${numBlendShapes} columns but frame specified only $numFrameWeights weights");
}
_data.addAll(frame
.skip(2)
.map((weight) => double.parse(weight))
.cast<double>()
.toList());
}
return Tuple2(header.skip(2).toList(), Float32List.fromList(_data));
}
}

13
lib/animations/live_link_face_bone_driver.dart
View File

@@ -1,13 +0,0 @@
import 'dart:math';
import 'package:polyvox_filament/animations/bone_driver.dart';
import 'package:vector_math/vector_math.dart';
BoneDriver getLiveLinkFaceBoneDrivers(String bone) {
return BoneDriver(bone, {
"HeadPitch":
Transformation(Quaternion.axisAngle(Vector3(0, 0, -1), pi / 3)),
"HeadRoll": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)),
"HeadYaw": Transformation(Quaternion.axisAngle(Vector3(0, 1, 0), pi / 2)),
});
}

4
lib/filament_controller.dart
View File

@@ -1,8 +1,8 @@
import 'dart:async'; import 'dart:async';
import 'dart:ui' as ui; import 'dart:ui' as ui;
import 'package:flutter/services.dart'; import 'package:flutter/services.dart';
import 'package:polyvox_filament/animations/bone_animation_data.dart';
import 'package:polyvox_filament/animations/morph_animation_data.dart'; import 'package:polyvox_filament/animations/animation_data.dart';
typedef FilamentEntity = int; typedef FilamentEntity = int;

4
lib/filament_controller_ffi.dart
View File

@@ -5,8 +5,8 @@ import 'dart:ui' as ui;
import 'package:flutter/services.dart'; import 'package:flutter/services.dart';
import 'package:ffi/ffi.dart'; import 'package:ffi/ffi.dart';
import 'package:polyvox_filament/filament_controller.dart'; import 'package:polyvox_filament/filament_controller.dart';
import 'package:polyvox_filament/animations/bone_animation_data.dart';
import 'package:polyvox_filament/animations/morph_animation_data.dart'; import 'package:polyvox_filament/animations/animation_data.dart';
import 'package:polyvox_filament/generated_bindings.dart'; import 'package:polyvox_filament/generated_bindings.dart';
const FilamentEntity _FILAMENT_ASSET_ERROR = 0; const FilamentEntity _FILAMENT_ASSET_ERROR = 0;

4
lib/filament_controller_method_channel.dart
View File

@@ -2,8 +2,8 @@ import 'dart:async';
import 'dart:ui' as ui; import 'dart:ui' as ui;
import 'package:flutter/services.dart'; import 'package:flutter/services.dart';
import 'package:polyvox_filament/filament_controller.dart'; import 'package:polyvox_filament/filament_controller.dart';
import 'package:polyvox_filament/animations/bone_animation_data.dart';
import 'package:polyvox_filament/animations/morph_animation_data.dart'; import 'package:polyvox_filament/animations/animation_data.dart';
import 'package:polyvox_filament/generated_bindings_web.dart'; import 'package:polyvox_filament/generated_bindings_web.dart';
import 'filament_controller.dart'; import 'filament_controller.dart';

70
test/bone_driver_test.dart
View File

@@ -1,70 +0,0 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:polyvox_filament/animations/bone_driver.dart';
import 'package:vector_math/vector_math.dart';
void main() {
group('BoneDriver', () {
test(
'transform should yield correct Quaternions for given morphTargetFrameData',
() {
final bone = 'bone1';
final transformations = {
'blendshape1': Transformation(Quaternion(1, 0, 0, 1)),
'blendshape2': Transformation(Quaternion(0, 1, 0, 1)),
};
final morphTargetFrameData = <String, List<double>>{
'blendshape1': [0.5, -0.5],
'blendshape2': [-1, 1],
};
final boneDriver = BoneDriver(bone, transformations);
final result = boneDriver.transform(morphTargetFrameData).toList();
expect(result.length, 2);
expect(result[0].x, -0.5);
expect(result[0].y, 0);
expect(result[0].z, -0.5);
expect(result[0].w, 0);
expect(result[1].x, 0.5);
expect(result[1].y, 0);
expect(result[1].z, 0.5);
expect(result[1].w, 0);
});
test(
'transform should throw AssertionError when morphTargetFrameData keys do not match transformations keys',
() {
final bone = 'bone1';
final transformations = {
'blendshape1': Transformation(Quaternion(1, 0, 0, 0)),
'blendshape2': Transformation(Quaternion(0, 1, 0, 0)),
};
final morphTargetFrameData = <String, List<double>>{
'blendshape1': [0.5, -0.5],
'blendshape3': [-1, 1],
};
final boneDriver = BoneDriver(bone, transformations);
expect(() => boneDriver.transform(morphTargetFrameData),
throwsA(isA<AssertionError>()));
});
test(
'transform should throw AssertionError when morphTargetFrameData values lengths do not match',
() {
final bone = 'bone1';
final transformations = {
'blendshape1': Transformation(Quaternion(1, 0, 0, 0)),
'blendshape2': Transformation(Quaternion(0, 1, 0, 0)),
};
final morphTargetFrameData = <String, List<double>>{
'blendshape1': [0.5, -0.5],
'blendshape2': [-1],
};
final boneDriver = BoneDriver(bone, transformations);
expect(() => boneDriver.transform(morphTargetFrameData),
throwsA(isA<AssertionError>()));
});
});
}