maobo/cup_edit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix dynamic bone animations

Browse Source
This commit is contained in:
Nick Fisher
2023-04-28 20:53:16 +08:00
parent 62c4be0563
commit bbb3cb5727
13 changed files with 170 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
example/lib/main.dart
View File

@@ -214,9 +214,10 @@ class _MyAppState extends State<MyApp> with SingleTickerProviderStateMixin {
return vals;
}).toList());
_filamentController.setBoneAnimation(_cube!, [
BoneAnimationData("Bone.001", "Cube.001", frameData, 1000.0 / 60.0)
]);
_filamentController.setBoneAnimation(
_cube!,
BoneAnimationData(
"Bone.001", ["Cube.001"], frameData, 1000.0 / 60.0));
// ,
// "Bone.001",
// "Cube.001",

3
ios/include/AssetManager.hpp
View File

@@ -54,7 +54,8 @@ namespace polyvox {
int numFrames,
int numBones,
const char** const boneNames,
const char* const meshName,
const char** const meshName,
int numMeshTargets,
float frameLengthInMs);
void playAnimation(EntityId e, int index, bool loop, bool reverse);
void stopAnimation(EntityId e, int index);

3
ios/include/PolyvoxFilamentApi.h
View File

@@ -64,7 +64,8 @@ void set_bone_animation(
int numFrames,
int numBones,
const char** const boneNames,
const char* const meshName,
const char** const meshName,
int numMeshTargets,
float frameLengthInMs);
void play_animation(void* assetManager, EntityId asset, int index, bool loop, bool reverse);

15
ios/include/SceneAsset.hpp
View File

@@ -22,7 +22,7 @@
extern "C" {
#include "PolyvoxFilamentApi.h"
}
template class std::vector<float>;
namespace polyvox {
using namespace filament;
using namespace filament::gltfio;
@@ -56,8 +56,12 @@ namespace polyvox {
// Multiple bones are supported but these must be skinned to a single mesh target.
//
struct BoneAnimationBuffer {
utils::Entity mMeshTarget;
vector<utils::Entity> mMeshTargets;
vector<uint8_t> mBones;
vector<math::mat4f> mBaseTransforms;
// vector<math::float3> mBaseTranslations; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this.
// vector<math::quatf> mBaseRotations; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this.
// vector<math::float3> mBaseScales; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this.
size_t skinIndex = 0;
int mNumFrames = -1;
float mFrameLengthInMs = 0;
@@ -99,10 +103,5 @@ namespace polyvox {
mAnimations[i].mDuration = mAnimator->getAnimationDuration(i);
}
}
};
}
}

168
ios/src/AssetManager.cpp
View File

@@ -1,16 +1,17 @@
#include "AssetManager.hpp"
#include "Log.hpp"
#include <thread>
#include <filament/Engine.h>
#include <filament/TransformManager.h>
#include <filament/Texture.h>
#include <filament/RenderableManager.h>
#include <gltfio/Animator.h>
#include <gltfio/AssetLoader.h>
#include <gltfio/FilamentAsset.h>
#include <gltfio/ResourceLoader.h>
#include <gltfio/TextureProvider.h>
#include <gltfio/math.h>
#include <imageio/ImageDecoder.h>
@@ -322,8 +323,10 @@ void AssetManager::updateAnimations() {
frameNumber
);
}
asset.mAnimator->updateBoneMatrices();
}
}
}
void AssetManager::setBoneTransform(SceneAsset& asset, int frameNumber) {
@@ -336,61 +339,30 @@ void AssetManager::updateAnimations() {
int skinIndex = 0;
math::mat4f inverseGlobalTransform = inverse(
transformManager.getWorldTransform(
transformManager.getInstance(asset.mBoneAnimationBuffer.mMeshTarget)
)
);
auto renderable = rm.getInstance(asset.mBoneAnimationBuffer.mMeshTarget);
for(int i = 0; i < asset.mBoneAnimationBuffer.mBones.size(); i++) {
auto mBoneIndex = asset.mBoneAnimationBuffer.mBones[i];
auto frameDataOffset = (frameNumber * asset.mBoneAnimationBuffer.mBones.size() * 7) + asset.mBoneAnimationBuffer.mBones[i];
auto frameDataOffset = (frameNumber * asset.mBoneAnimationBuffer.mBones.size() * 7) + (i * 7);
utils::Entity joint = filamentInstance->getJointsAt(skinIndex)[mBoneIndex];
if(joint.isNull()) {
Log("ERROR : joint not found");
continue;
}
// RenderableManager::Bone bone { math::quatf{
// asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6],
// asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3],
// asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4],
// asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5]
// },
// math::float3 {
// asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+0],
// asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+1],
// asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+2]
// }
// };
// rm.setBones(
// renderable,
// &bone,
// 1,
// mBoneIndex
// );
const math::mat4f localTransform(math::quatf{
asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3],
asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4],
asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5],
asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6]
vector<float>& fd = asset.mBoneAnimationBuffer.mFrameData;
math::mat4f localTransform(math::quatf {
fd[frameDataOffset+3],
fd[frameDataOffset+4],
fd[frameDataOffset+5],
fd[frameDataOffset+6],
});
const math::mat4f& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(skinIndex)[mBoneIndex];
auto jointInstance = transformManager.getInstance(joint);
math::mat4f globalJointTransform = transformManager.getWorldTransform(jointInstance);
math::mat4f boneTransform = inverseGlobalTransform * globalJointTransform * inverseBindMatrix * localTransform;
rm.setBones(
renderable,
&boneTransform,
1,
mBoneIndex
);
auto xform = asset.mBoneAnimationBuffer.mBaseTransforms[i];
transformManager.setTransform(jointInstance, xform * localTransform);
}
}
@@ -486,22 +458,18 @@ bool AssetManager::setBoneAnimationBuffer(
int numFrames,
int numBones,
const char** const boneNames,
const char* const meshName,
const char** const meshNames,
int numMeshTargets,
float frameLengthInMs) {
const auto& pos = _entityIdLookup.find(entityId);
if(pos == _entityIdLookup.end()) {
Log("ERROR: asset not found for entity.");
return false;
}
auto& asset = _assets[pos->second];
auto entity = findEntityByName(asset, meshName);
if(!entity) {
Log("Mesh target %s for bone animation could not be found", meshName);
return false;
}
auto filamentInstance = asset.mAsset->getInstance();
size_t skinCount = filamentInstance->getSkinCount();
@@ -510,46 +478,71 @@ bool AssetManager::setBoneAnimationBuffer(
Log("WARNING - skin count > 1 not currently implemented. This will probably not work");
}
TransformManager &transformManager = _engine->getTransformManager();
int skinIndex = 0;
const utils::Entity* joints = filamentInstance->getJointsAt(skinIndex);
size_t numJoints = filamentInstance->getJointCountAt(skinIndex);
asset.mBoneAnimationBuffer.mBones.clear();
BoneAnimationBuffer& animationBuffer = asset.mBoneAnimationBuffer;
// if an animation has already been set, reset the transform for the respective bones
for(int i = 0; i < animationBuffer.mBones.size(); i++) {
auto boneIndex = animationBuffer.mBones[i];
auto jointInstance = transformManager.getInstance(joints[boneIndex]);
transformManager.setTransform(jointInstance, animationBuffer.mBaseTransforms[i]);
}
asset.mAnimator->resetBoneMatrices();
animationBuffer.mBones.resize(numBones);
animationBuffer.mBaseTransforms.resize(numBones);
for(int i = 0; i < numBones; i++) {
Log("Bone %s", boneNames[i]);
for(int j = 0; j < numJoints; j++) {
const char* jointName = _ncm->getName(_ncm->getInstance(joints[j]));
if(strcmp(jointName, boneNames[i]) == 0) {
asset.mBoneAnimationBuffer.mBones.push_back(j);
auto jointInstance = transformManager.getInstance(joints[j]);
// auto currentXform = ;
auto baseTransform = transformManager.getTransform(jointInstance); // inverse(filamentInstance->getInverseBindMatricesAt(skinIndex)[j]);
animationBuffer.mBaseTransforms[i] = baseTransform;
animationBuffer.mBones[i] = j;
break;
}
}
}
if(asset.mBoneAnimationBuffer.mBones.size() != numBones) {
if(animationBuffer.mBones.size() != numBones) {
Log("Failed to find one or more bone indices");
return false;
}
asset.mBoneAnimationBuffer.mFrameData.clear();
animationBuffer.mFrameData.clear();
// 7 == locX, locY, locZ, rotW, rotX, rotY, rotZ
asset.mBoneAnimationBuffer.mFrameData.resize(numFrames * numBones * 7);
asset.mBoneAnimationBuffer.mFrameData.insert(
asset.mBoneAnimationBuffer.mFrameData.begin(),
animationBuffer.mFrameData.resize(numFrames * numBones * 7);
animationBuffer.mFrameData.insert(
animationBuffer.mFrameData.begin(),
frameData,
frameData + numFrames * numBones * 7
);
Log("%d frames for %d bones", numFrames, numBones);
// for(int i = 0; i < numFrames * numBones * 7; i++) {
// Log("Frame data @ %d is %f", i, frameData[i]);
// }
asset.mBoneAnimationBuffer.mFrameLengthInMs = frameLengthInMs;
asset.mBoneAnimationBuffer.mNumFrames = numFrames;
animationBuffer.mFrameLengthInMs = frameLengthInMs;
animationBuffer.mNumFrames = numFrames;
asset.mBoneAnimationBuffer.mMeshTarget = entity;
animationBuffer.mMeshTargets.clear();
for(int i = 0; i < numMeshTargets; i++) {
auto entity = findEntityByName(asset, meshNames[i]);
if(!entity) {
Log("Mesh target %s for bone animation could not be found", meshNames[i]);
return false;
}
Log("Added mesh target %s", meshNames[i]);
animationBuffer.mMeshTargets.push_back(entity);
}
auto& animation = asset.mAnimations[asset.mAnimations.size() - 1];
animation.mStart = std::chrono::high_resolution_clock::now();
animation.mAnimating = true;
@@ -557,12 +550,6 @@ bool AssetManager::setBoneAnimationBuffer(
animation.mDuration = (frameLengthInMs * numFrames) / 1000.0f;
asset.mAnimating = true;
// // Log(", set start to %f and duration to %f", );
// Log("Successfully set bone animation buffer, set start to %d, dur is %f",
// std::chrono::duration_cast<std::chrono::milliseconds>(asset.mAnimations[1].mStart.time_since_epoch()).count(),
// asset.mAnimations[1].mDuration
// );
return true;
}
@@ -833,3 +820,40 @@ size_t AssetManager::getLightEntityCount(EntityId entity) const noexcept {
} // namespace polyvox
// auto& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(skinIndex)[mBoneIndex];
// auto globalJointTransform = transformManager.getWorldTransform(jointInstance);
// for(auto& target : asset.mBoneAnimationBuffer.mMeshTargets) {
// auto inverseGlobalTransform = inverse(
// transformManager.getWorldTransform(
// transformManager.getInstance(target)
// )
// );
// auto boneTransform = inverseGlobalTransform * globalJointTransform * localTransform * inverseBindMatrix;
// auto renderable = rm.getInstance(target);
// rm.setBones(
// renderable,
// &boneTransform,
// 1,
// mBoneIndex
// );
// }
// 1.0f, 0.0f, 0.0f, 0.0f,
// 0.0f, 0.0f, 1.0f, 0.0f,
// 0.0f, -1.0f, 0.0f, 0.0f,
// 0.0f, 0.0f, 0.0f, 1.0f
// };
// Log("TRANSFORM");
// Log("%f %f %f %f", localTransform[0][0], localTransform[1][0], localTransform[2][0], localTransform[3][0] ) ;
// Log("%f %f %f %f", localTransform[0][1], localTransform[1][1], localTransform[2][1], localTransform[3][1] ) ;
// Log("%f %f %f %f", localTransform[0][2], localTransform[1][2], localTransform[2][2], localTransform[3][2] ) ;
// Log("%f %f %f %f", localTransform[0][3], localTransform[1][3], localTransform[2][3], localTransform[3][3] ) ;
// transformManager.getTransform(jointInstance);

2
ios/src/FilamentViewer.cpp
View File

@@ -788,7 +788,7 @@ void FilamentViewer::render(uint64_t frameTimeInNanos) {
}
if(_frameCount == 60) {
Log("1 sec average for asset animation update %f", _elapsed);
Log("1 sec average for asset animation update %f", _elapsed / 60);
_elapsed = 0;
_frameCount = 0;
}

6
ios/src/PolyvoxFilamentApi.cpp
View File

@@ -362,7 +362,8 @@ extern "C" {
int numFrames,
int numBones,
const char** const boneNames,
const char* const meshName,
const char** const meshNames,
int numMeshTargets,
float frameLengthInMs) {
//std::packaged_task<void()> lambda([=]() mutable {
((AssetManager*)assetManager)->setBoneAnimationBuffer(
@@ -371,7 +372,8 @@ extern "C" {
numFrames,
numBones,
boneNames,
meshName,
meshNames,
numMeshTargets,
frameLengthInMs
);
//});

4
lib/animations/bone_animation_data.dart
View File

@@ -9,9 +9,9 @@ import 'package:vector_math/vector_math.dart';
///
class BoneAnimationData {
final String boneName;
final String meshName;
final List<String> meshNames;
final Float32List frameData;
double frameLengthInMs;
BoneAnimationData(
this.boneName, this.meshName, this.frameData, this.frameLengthInMs);
this.boneName, this.meshNames, this.frameData, this.frameLengthInMs);
}

14
lib/animations/bone_driver.dart
View File

@@ -46,11 +46,21 @@ class BoneDriver {
rotation.x *= weight;
rotation.y *= weight;
rotation.z *= weight;
rotation.w = 1;
return rotation;
}).toList();
yield rotations.fold(
rotations.first, (Quaternion a, Quaternion b) => a * b);
if (frameNum == 0) {
print(rotations);
}
var result = rotations.fold(
rotations.first, (Quaternion a, Quaternion b) => a + b);
result.w = 1;
print("RESULT $result");
yield result;
// .normalized();
// todo - bone translations
}
}

16
lib/animations/csv_animation.dart
View File

@@ -11,11 +11,12 @@ import 'package:vector_math/vector_math.dart';
/// A class for loading animation data from a single CSV and allocating between morph/bone animation with help.
///
class DynamicAnimation {
final MorphAnimationData morphAnimation;
final MorphAnimationData? morphAnimation;
final List<BoneAnimationData> boneAnimation;
factory DynamicAnimation.load(String meshName, String csvPath,
factory DynamicAnimation.load(String? meshName, String csvPath,
{List<BoneDriver>? boneDrivers,
List<String>? boneMeshes,
String? boneDriverConfigPath,
double? framerate}) {
// create a MorphAnimationData instance from the given CSV
@@ -23,8 +24,9 @@ class DynamicAnimation {
var frameLengthInMs = 1000 / (framerate ?? 60.0);
var morphNames = llf
.item1; //.where((name) => !boneDrivers.any((element) => element.blendshape == name));
var morphAnimationData =
MorphAnimationData(meshName, llf.item2, morphNames, frameLengthInMs);
var morphAnimationData = MorphAnimationData(
meshName ?? "NULL", llf.item2, morphNames, frameLengthInMs);
final boneAnimations = <BoneAnimationData>[];
@@ -45,14 +47,14 @@ class DynamicAnimation {
// iterate over every bone driver
if (boneDrivers != null) {
for (var driver in boneDrivers) {
// get all frames for the single the blendshape
// collect the frame data for the blendshapes that this driver uses
var morphData = driver.transformations
.map((String blendshape, Transformation transformation) {
return MapEntry(
blendshape, morphAnimationData.getData(blendshape).toList());
});
// apply the driver to the blendshape weight
// apply the driver to the frame data
var transformedQ = driver.transform(morphData).toList();
// transform the quaternion to a Float32List
@@ -60,7 +62,7 @@ class DynamicAnimation {
// add to the list of boneAnimations
boneAnimations.add(BoneAnimationData(
driver.bone, meshName, transformedF, frameLengthInMs));
driver.bone, boneMeshes!, transformedF, frameLengthInMs));
}
}

5
lib/animations/live_link_face_bone_driver.dart
View File

@@ -5,8 +5,9 @@ import 'package:vector_math/vector_math.dart';
BoneDriver getLiveLinkFaceBoneDrivers(String bone) {
return BoneDriver(bone, {
"HeadPitch": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)),
"HeadRoll": Transformation(Quaternion.axisAngle(Vector3(0, 0, 1), pi / 2)),
"HeadPitch":
Transformation(Quaternion.axisAngle(Vector3(0, 0, -1), pi / 3)),
"HeadRoll": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)),
"HeadYaw": Transformation(Quaternion.axisAngle(Vector3(0, 1, 0), pi / 2)),
});
}

47
lib/filament_controller.dart
View File

@@ -320,32 +320,26 @@ class FilamentController {
/// Animates morph target weights/bone transforms (where each frame requires a duration of [frameLengthInMs].
/// [morphWeights] is a list of doubles in frame-major format.
/// Each frame is [numWeights] in length, and each entry is the weight to be applied to the morph target located at that index in the mesh primitive at that frame.
/// for now we only allow animating a single bone (though multiple skinned targets are supported)
///
void setBoneAnimation(
FilamentEntity asset, List<BoneAnimationData> animations) async {
// for future compatibility, instances of BoneAnimationData can specify individual mesh targets
// however on the rendering side we currently only allow one set of frame data for one mesh target (though multiple bones are supported).
// this is a check that all animations are targeting the same mesh
assert(animations.map((e) => e.meshName).toSet().length == 1);
var data =
calloc<Float>(animations.length * animations.first.frameData.length);
FilamentEntity asset, BoneAnimationData animation) async {
var data = calloc<Float>(animation.frameData.length);
int offset = 0;
var numFrames = animations.first.frameData.length ~/ 7;
var boneNames = calloc<Pointer<Char>>(animations.length);
int animIdx = 0;
for (var animation in animations) {
if (animation.frameData.length ~/ 7 != numFrames) {
throw Exception(
"All bone animations must share the same animation frame data length.");
}
for (int i = 0; i < animation.frameData.length; i++) {
data.elementAt(offset).value = animation.frameData[i];
offset += 1;
}
boneNames.elementAt(animIdx).value =
animation.boneName.toNativeUtf8().cast<Char>();
animIdx++;
var numFrames = animation.frameData.length ~/ 7;
var boneNames = calloc<Pointer<Char>>(1);
boneNames.elementAt(0).value =
animation.boneName.toNativeUtf8().cast<Char>();
var meshNames = calloc<Pointer<Char>>(animation.meshNames.length);
for (int i = 0; i < animation.meshNames.length; i++) {
meshNames.elementAt(i).value =
animation.meshNames[i].toNativeUtf8().cast<Char>();
}
for (int i = 0; i < animation.frameData.length; i++) {
data.elementAt(offset).value = animation.frameData[i];
offset += 1;
}
_nativeLibrary.set_bone_animation(
@@ -353,10 +347,11 @@ class FilamentController {
asset,
data,
numFrames,
animations.length,
1,
boneNames,
animations.first.meshName.toNativeUtf8().cast<Char>(),
animations.first.frameLengthInMs);
meshNames,
animation.meshNames.length,
animation.frameLengthInMs);
calloc.free(data);
}

10
lib/generated_bindings.dart
View File

@@ -659,7 +659,8 @@ class NativeLibrary {
int numFrames,
int numBones,
ffi.Pointer<ffi.Pointer<ffi.Char>> boneNames,
ffi.Pointer<ffi.Char> meshName,
ffi.Pointer<ffi.Pointer<ffi.Char>> meshName,
int numMeshTargets,
double frameLengthInMs,
) {
return _set_bone_animation(
@@ -670,6 +671,7 @@ class NativeLibrary {
numBones,
boneNames,
meshName,
numMeshTargets,
frameLengthInMs,
);
}
@@ -683,7 +685,8 @@ class NativeLibrary {
ffi.Int,
ffi.Int,
ffi.Pointer<ffi.Pointer<ffi.Char>>,
ffi.Pointer<ffi.Char>,
ffi.Pointer<ffi.Pointer<ffi.Char>>,
ffi.Int,
ffi.Float)>>('set_bone_animation');
late final _set_bone_animation = _set_bone_animationPtr.asFunction<
void Function(
@@ -693,7 +696,8 @@ class NativeLibrary {
int,
int,
ffi.Pointer<ffi.Pointer<ffi.Char>>,
ffi.Pointer<ffi.Char>,
ffi.Pointer<ffi.Pointer<ffi.Char>>,
int,
double)>();
void play_animation(