diff --git a/example/lib/main.dart b/example/lib/main.dart index db424975..bad4a0a2 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -214,9 +214,10 @@ class _MyAppState extends State with SingleTickerProviderStateMixin { return vals; }).toList()); - _filamentController.setBoneAnimation(_cube!, [ - BoneAnimationData("Bone.001", "Cube.001", frameData, 1000.0 / 60.0) - ]); + _filamentController.setBoneAnimation( + _cube!, + BoneAnimationData( + "Bone.001", ["Cube.001"], frameData, 1000.0 / 60.0)); // , // "Bone.001", // "Cube.001", diff --git a/ios/include/AssetManager.hpp b/ios/include/AssetManager.hpp index 7b2da476..73333b6b 100644 --- a/ios/include/AssetManager.hpp +++ b/ios/include/AssetManager.hpp @@ -54,7 +54,8 @@ namespace polyvox { int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshName, + int numMeshTargets, float frameLengthInMs); void playAnimation(EntityId e, int index, bool loop, bool reverse); void stopAnimation(EntityId e, int index); diff --git a/ios/include/PolyvoxFilamentApi.h b/ios/include/PolyvoxFilamentApi.h index 8f92756f..2739f4e8 100644 --- a/ios/include/PolyvoxFilamentApi.h +++ b/ios/include/PolyvoxFilamentApi.h @@ -64,7 +64,8 @@ void set_bone_animation( int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshName, + int numMeshTargets, float frameLengthInMs); void play_animation(void* assetManager, EntityId asset, int index, bool loop, bool reverse); diff --git a/ios/include/SceneAsset.hpp b/ios/include/SceneAsset.hpp index 1eb93924..d7ee58e5 100644 --- a/ios/include/SceneAsset.hpp +++ b/ios/include/SceneAsset.hpp @@ -22,7 +22,7 @@ extern "C" { #include "PolyvoxFilamentApi.h" } - +template class std::vector; namespace polyvox { using namespace filament; using namespace filament::gltfio; @@ -56,8 +56,12 @@ namespace polyvox { // Multiple bones are supported but these must be skinned to a single mesh target. // struct BoneAnimationBuffer { - utils::Entity mMeshTarget; + vector mMeshTargets; vector mBones; + vector mBaseTransforms; + // vector mBaseTranslations; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this. + // vector mBaseRotations; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this. + // vector mBaseScales; // these are the base transforms for the bones we will animate; the translations/rotations in mFrameData will be relative to this. size_t skinIndex = 0; int mNumFrames = -1; float mFrameLengthInMs = 0; @@ -99,10 +103,5 @@ namespace polyvox { mAnimations[i].mDuration = mAnimator->getAnimationDuration(i); } } - - }; - -} - - \ No newline at end of file +} \ No newline at end of file diff --git a/ios/src/AssetManager.cpp b/ios/src/AssetManager.cpp index 5c1f83cb..1dcbc962 100644 --- a/ios/src/AssetManager.cpp +++ b/ios/src/AssetManager.cpp @@ -1,16 +1,17 @@ #include "AssetManager.hpp" -#include "Log.hpp" #include #include #include #include #include + #include #include #include #include #include +#include #include @@ -322,8 +323,10 @@ void AssetManager::updateAnimations() { frameNumber ); } + asset.mAnimator->updateBoneMatrices(); } } + } void AssetManager::setBoneTransform(SceneAsset& asset, int frameNumber) { @@ -336,61 +339,30 @@ void AssetManager::updateAnimations() { int skinIndex = 0; - math::mat4f inverseGlobalTransform = inverse( - transformManager.getWorldTransform( - transformManager.getInstance(asset.mBoneAnimationBuffer.mMeshTarget) - ) - ); - - auto renderable = rm.getInstance(asset.mBoneAnimationBuffer.mMeshTarget); - for(int i = 0; i < asset.mBoneAnimationBuffer.mBones.size(); i++) { auto mBoneIndex = asset.mBoneAnimationBuffer.mBones[i]; - auto frameDataOffset = (frameNumber * asset.mBoneAnimationBuffer.mBones.size() * 7) + asset.mBoneAnimationBuffer.mBones[i]; + auto frameDataOffset = (frameNumber * asset.mBoneAnimationBuffer.mBones.size() * 7) + (i * 7); utils::Entity joint = filamentInstance->getJointsAt(skinIndex)[mBoneIndex]; if(joint.isNull()) { Log("ERROR : joint not found"); continue; } - // RenderableManager::Bone bone { math::quatf{ - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5] - // }, - // math::float3 { - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+0], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+1], - // asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+2] - // } - - // }; - // rm.setBones( - // renderable, - // &bone, - // 1, - // mBoneIndex - // ); - const math::mat4f localTransform(math::quatf{ - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+3], - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+4], - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+5], - asset.mBoneAnimationBuffer.mFrameData[frameDataOffset+6] + + vector& fd = asset.mBoneAnimationBuffer.mFrameData; + + math::mat4f localTransform(math::quatf { + fd[frameDataOffset+3], + fd[frameDataOffset+4], + fd[frameDataOffset+5], + fd[frameDataOffset+6], }); - const math::mat4f& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(skinIndex)[mBoneIndex]; + auto jointInstance = transformManager.getInstance(joint); - math::mat4f globalJointTransform = transformManager.getWorldTransform(jointInstance); - - math::mat4f boneTransform = inverseGlobalTransform * globalJointTransform * inverseBindMatrix * localTransform; - rm.setBones( - renderable, - &boneTransform, - 1, - mBoneIndex - ); + auto xform = asset.mBoneAnimationBuffer.mBaseTransforms[i]; + transformManager.setTransform(jointInstance, xform * localTransform); } } @@ -486,22 +458,18 @@ bool AssetManager::setBoneAnimationBuffer( int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshNames, + int numMeshTargets, float frameLengthInMs) { + + const auto& pos = _entityIdLookup.find(entityId); if(pos == _entityIdLookup.end()) { Log("ERROR: asset not found for entity."); return false; } auto& asset = _assets[pos->second]; - - auto entity = findEntityByName(asset, meshName); - if(!entity) { - Log("Mesh target %s for bone animation could not be found", meshName); - return false; - } - auto filamentInstance = asset.mAsset->getInstance(); size_t skinCount = filamentInstance->getSkinCount(); @@ -510,46 +478,71 @@ bool AssetManager::setBoneAnimationBuffer( Log("WARNING - skin count > 1 not currently implemented. This will probably not work"); } + TransformManager &transformManager = _engine->getTransformManager(); + int skinIndex = 0; const utils::Entity* joints = filamentInstance->getJointsAt(skinIndex); size_t numJoints = filamentInstance->getJointCountAt(skinIndex); - asset.mBoneAnimationBuffer.mBones.clear(); + BoneAnimationBuffer& animationBuffer = asset.mBoneAnimationBuffer; + + // if an animation has already been set, reset the transform for the respective bones + for(int i = 0; i < animationBuffer.mBones.size(); i++) { + auto boneIndex = animationBuffer.mBones[i]; + auto jointInstance = transformManager.getInstance(joints[boneIndex]); + transformManager.setTransform(jointInstance, animationBuffer.mBaseTransforms[i]); + } + + asset.mAnimator->resetBoneMatrices(); + + animationBuffer.mBones.resize(numBones); + animationBuffer.mBaseTransforms.resize(numBones); + for(int i = 0; i < numBones; i++) { Log("Bone %s", boneNames[i]); for(int j = 0; j < numJoints; j++) { const char* jointName = _ncm->getName(_ncm->getInstance(joints[j])); if(strcmp(jointName, boneNames[i]) == 0) { - asset.mBoneAnimationBuffer.mBones.push_back(j); + auto jointInstance = transformManager.getInstance(joints[j]); + // auto currentXform = ; + auto baseTransform = transformManager.getTransform(jointInstance); // inverse(filamentInstance->getInverseBindMatricesAt(skinIndex)[j]); + animationBuffer.mBaseTransforms[i] = baseTransform; + animationBuffer.mBones[i] = j; break; } } } - if(asset.mBoneAnimationBuffer.mBones.size() != numBones) { + if(animationBuffer.mBones.size() != numBones) { Log("Failed to find one or more bone indices"); return false; } - asset.mBoneAnimationBuffer.mFrameData.clear(); + animationBuffer.mFrameData.clear(); // 7 == locX, locY, locZ, rotW, rotX, rotY, rotZ - asset.mBoneAnimationBuffer.mFrameData.resize(numFrames * numBones * 7); - asset.mBoneAnimationBuffer.mFrameData.insert( - asset.mBoneAnimationBuffer.mFrameData.begin(), + animationBuffer.mFrameData.resize(numFrames * numBones * 7); + animationBuffer.mFrameData.insert( + animationBuffer.mFrameData.begin(), frameData, frameData + numFrames * numBones * 7 ); Log("%d frames for %d bones", numFrames, numBones); - // for(int i = 0; i < numFrames * numBones * 7; i++) { - // Log("Frame data @ %d is %f", i, frameData[i]); - // } - - asset.mBoneAnimationBuffer.mFrameLengthInMs = frameLengthInMs; - asset.mBoneAnimationBuffer.mNumFrames = numFrames; + animationBuffer.mFrameLengthInMs = frameLengthInMs; + animationBuffer.mNumFrames = numFrames; - asset.mBoneAnimationBuffer.mMeshTarget = entity; + animationBuffer.mMeshTargets.clear(); + for(int i = 0; i < numMeshTargets; i++) { + auto entity = findEntityByName(asset, meshNames[i]); + if(!entity) { + Log("Mesh target %s for bone animation could not be found", meshNames[i]); + return false; + } + Log("Added mesh target %s", meshNames[i]); + animationBuffer.mMeshTargets.push_back(entity); + } + auto& animation = asset.mAnimations[asset.mAnimations.size() - 1]; animation.mStart = std::chrono::high_resolution_clock::now(); animation.mAnimating = true; @@ -557,12 +550,6 @@ bool AssetManager::setBoneAnimationBuffer( animation.mDuration = (frameLengthInMs * numFrames) / 1000.0f; asset.mAnimating = true; - // // Log(", set start to %f and duration to %f", ); - // Log("Successfully set bone animation buffer, set start to %d, dur is %f", - // std::chrono::duration_cast(asset.mAnimations[1].mStart.time_since_epoch()).count(), - // asset.mAnimations[1].mDuration - // ); - return true; } @@ -833,3 +820,40 @@ size_t AssetManager::getLightEntityCount(EntityId entity) const noexcept { } // namespace polyvox + + +// auto& inverseBindMatrix = filamentInstance->getInverseBindMatricesAt(skinIndex)[mBoneIndex]; + + // auto globalJointTransform = transformManager.getWorldTransform(jointInstance); + + // for(auto& target : asset.mBoneAnimationBuffer.mMeshTargets) { + + // auto inverseGlobalTransform = inverse( + // transformManager.getWorldTransform( + // transformManager.getInstance(target) + // ) + // ); + + // auto boneTransform = inverseGlobalTransform * globalJointTransform * localTransform * inverseBindMatrix; + // auto renderable = rm.getInstance(target); + // rm.setBones( + // renderable, + // &boneTransform, + // 1, + // mBoneIndex + // ); + // } + + + + // 1.0f, 0.0f, 0.0f, 0.0f, + // 0.0f, 0.0f, 1.0f, 0.0f, + // 0.0f, -1.0f, 0.0f, 0.0f, + // 0.0f, 0.0f, 0.0f, 1.0f + // }; + // Log("TRANSFORM"); + // Log("%f %f %f %f", localTransform[0][0], localTransform[1][0], localTransform[2][0], localTransform[3][0] ) ; + // Log("%f %f %f %f", localTransform[0][1], localTransform[1][1], localTransform[2][1], localTransform[3][1] ) ; + // Log("%f %f %f %f", localTransform[0][2], localTransform[1][2], localTransform[2][2], localTransform[3][2] ) ; + // Log("%f %f %f %f", localTransform[0][3], localTransform[1][3], localTransform[2][3], localTransform[3][3] ) ; + // transformManager.getTransform(jointInstance); \ No newline at end of file diff --git a/ios/src/FilamentViewer.cpp b/ios/src/FilamentViewer.cpp index 6fdab983..b1e98329 100644 --- a/ios/src/FilamentViewer.cpp +++ b/ios/src/FilamentViewer.cpp @@ -788,7 +788,7 @@ void FilamentViewer::render(uint64_t frameTimeInNanos) { } if(_frameCount == 60) { - Log("1 sec average for asset animation update %f", _elapsed); + Log("1 sec average for asset animation update %f", _elapsed / 60); _elapsed = 0; _frameCount = 0; } diff --git a/ios/src/PolyvoxFilamentApi.cpp b/ios/src/PolyvoxFilamentApi.cpp index be0cdff5..6aa69400 100644 --- a/ios/src/PolyvoxFilamentApi.cpp +++ b/ios/src/PolyvoxFilamentApi.cpp @@ -362,7 +362,8 @@ extern "C" { int numFrames, int numBones, const char** const boneNames, - const char* const meshName, + const char** const meshNames, + int numMeshTargets, float frameLengthInMs) { //std::packaged_task lambda([=]() mutable { ((AssetManager*)assetManager)->setBoneAnimationBuffer( @@ -371,7 +372,8 @@ extern "C" { numFrames, numBones, boneNames, - meshName, + meshNames, + numMeshTargets, frameLengthInMs ); //}); diff --git a/lib/animations/bone_animation_data.dart b/lib/animations/bone_animation_data.dart index 7bf9b7e3..672c7661 100644 --- a/lib/animations/bone_animation_data.dart +++ b/lib/animations/bone_animation_data.dart @@ -9,9 +9,9 @@ import 'package:vector_math/vector_math.dart'; /// class BoneAnimationData { final String boneName; - final String meshName; + final List meshNames; final Float32List frameData; double frameLengthInMs; BoneAnimationData( - this.boneName, this.meshName, this.frameData, this.frameLengthInMs); + this.boneName, this.meshNames, this.frameData, this.frameLengthInMs); } diff --git a/lib/animations/bone_driver.dart b/lib/animations/bone_driver.dart index e404efca..3ca30f5f 100644 --- a/lib/animations/bone_driver.dart +++ b/lib/animations/bone_driver.dart @@ -46,11 +46,21 @@ class BoneDriver { rotation.x *= weight; rotation.y *= weight; rotation.z *= weight; + rotation.w = 1; + return rotation; }).toList(); - yield rotations.fold( - rotations.first, (Quaternion a, Quaternion b) => a * b); + if (frameNum == 0) { + print(rotations); + } + + var result = rotations.fold( + rotations.first, (Quaternion a, Quaternion b) => a + b); + result.w = 1; + print("RESULT $result"); + yield result; + // .normalized(); // todo - bone translations } } diff --git a/lib/animations/csv_animation.dart b/lib/animations/csv_animation.dart index 1b9182e5..93b917bf 100644 --- a/lib/animations/csv_animation.dart +++ b/lib/animations/csv_animation.dart @@ -11,11 +11,12 @@ import 'package:vector_math/vector_math.dart'; /// A class for loading animation data from a single CSV and allocating between morph/bone animation with help. /// class DynamicAnimation { - final MorphAnimationData morphAnimation; + final MorphAnimationData? morphAnimation; final List boneAnimation; - factory DynamicAnimation.load(String meshName, String csvPath, + factory DynamicAnimation.load(String? meshName, String csvPath, {List? boneDrivers, + List? boneMeshes, String? boneDriverConfigPath, double? framerate}) { // create a MorphAnimationData instance from the given CSV @@ -23,8 +24,9 @@ class DynamicAnimation { var frameLengthInMs = 1000 / (framerate ?? 60.0); var morphNames = llf .item1; //.where((name) => !boneDrivers.any((element) => element.blendshape == name)); - var morphAnimationData = - MorphAnimationData(meshName, llf.item2, morphNames, frameLengthInMs); + + var morphAnimationData = MorphAnimationData( + meshName ?? "NULL", llf.item2, morphNames, frameLengthInMs); final boneAnimations = []; @@ -45,14 +47,14 @@ class DynamicAnimation { // iterate over every bone driver if (boneDrivers != null) { for (var driver in boneDrivers) { - // get all frames for the single the blendshape + // collect the frame data for the blendshapes that this driver uses var morphData = driver.transformations .map((String blendshape, Transformation transformation) { return MapEntry( blendshape, morphAnimationData.getData(blendshape).toList()); }); - // apply the driver to the blendshape weight + // apply the driver to the frame data var transformedQ = driver.transform(morphData).toList(); // transform the quaternion to a Float32List @@ -60,7 +62,7 @@ class DynamicAnimation { // add to the list of boneAnimations boneAnimations.add(BoneAnimationData( - driver.bone, meshName, transformedF, frameLengthInMs)); + driver.bone, boneMeshes!, transformedF, frameLengthInMs)); } } diff --git a/lib/animations/live_link_face_bone_driver.dart b/lib/animations/live_link_face_bone_driver.dart index b5cc9c99..921b8583 100644 --- a/lib/animations/live_link_face_bone_driver.dart +++ b/lib/animations/live_link_face_bone_driver.dart @@ -5,8 +5,9 @@ import 'package:vector_math/vector_math.dart'; BoneDriver getLiveLinkFaceBoneDrivers(String bone) { return BoneDriver(bone, { - "HeadPitch": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)), - "HeadRoll": Transformation(Quaternion.axisAngle(Vector3(0, 0, 1), pi / 2)), + "HeadPitch": + Transformation(Quaternion.axisAngle(Vector3(0, 0, -1), pi / 3)), + "HeadRoll": Transformation(Quaternion.axisAngle(Vector3(1, 0, 0), pi / 2)), "HeadYaw": Transformation(Quaternion.axisAngle(Vector3(0, 1, 0), pi / 2)), }); } diff --git a/lib/filament_controller.dart b/lib/filament_controller.dart index 52df302b..561862a9 100644 --- a/lib/filament_controller.dart +++ b/lib/filament_controller.dart @@ -320,32 +320,26 @@ class FilamentController { /// Animates morph target weights/bone transforms (where each frame requires a duration of [frameLengthInMs]. /// [morphWeights] is a list of doubles in frame-major format. /// Each frame is [numWeights] in length, and each entry is the weight to be applied to the morph target located at that index in the mesh primitive at that frame. + /// for now we only allow animating a single bone (though multiple skinned targets are supported) /// void setBoneAnimation( - FilamentEntity asset, List animations) async { - // for future compatibility, instances of BoneAnimationData can specify individual mesh targets - // however on the rendering side we currently only allow one set of frame data for one mesh target (though multiple bones are supported). - // this is a check that all animations are targeting the same mesh - assert(animations.map((e) => e.meshName).toSet().length == 1); - - var data = - calloc(animations.length * animations.first.frameData.length); + FilamentEntity asset, BoneAnimationData animation) async { + var data = calloc(animation.frameData.length); int offset = 0; - var numFrames = animations.first.frameData.length ~/ 7; - var boneNames = calloc>(animations.length); - int animIdx = 0; - for (var animation in animations) { - if (animation.frameData.length ~/ 7 != numFrames) { - throw Exception( - "All bone animations must share the same animation frame data length."); - } - for (int i = 0; i < animation.frameData.length; i++) { - data.elementAt(offset).value = animation.frameData[i]; - offset += 1; - } - boneNames.elementAt(animIdx).value = - animation.boneName.toNativeUtf8().cast(); - animIdx++; + var numFrames = animation.frameData.length ~/ 7; + var boneNames = calloc>(1); + boneNames.elementAt(0).value = + animation.boneName.toNativeUtf8().cast(); + + var meshNames = calloc>(animation.meshNames.length); + for (int i = 0; i < animation.meshNames.length; i++) { + meshNames.elementAt(i).value = + animation.meshNames[i].toNativeUtf8().cast(); + } + + for (int i = 0; i < animation.frameData.length; i++) { + data.elementAt(offset).value = animation.frameData[i]; + offset += 1; } _nativeLibrary.set_bone_animation( @@ -353,10 +347,11 @@ class FilamentController { asset, data, numFrames, - animations.length, + 1, boneNames, - animations.first.meshName.toNativeUtf8().cast(), - animations.first.frameLengthInMs); + meshNames, + animation.meshNames.length, + animation.frameLengthInMs); calloc.free(data); } diff --git a/lib/generated_bindings.dart b/lib/generated_bindings.dart index 2197eb07..1f807909 100644 --- a/lib/generated_bindings.dart +++ b/lib/generated_bindings.dart @@ -659,7 +659,8 @@ class NativeLibrary { int numFrames, int numBones, ffi.Pointer> boneNames, - ffi.Pointer meshName, + ffi.Pointer> meshName, + int numMeshTargets, double frameLengthInMs, ) { return _set_bone_animation( @@ -670,6 +671,7 @@ class NativeLibrary { numBones, boneNames, meshName, + numMeshTargets, frameLengthInMs, ); } @@ -683,7 +685,8 @@ class NativeLibrary { ffi.Int, ffi.Int, ffi.Pointer>, - ffi.Pointer, + ffi.Pointer>, + ffi.Int, ffi.Float)>>('set_bone_animation'); late final _set_bone_animation = _set_bone_animationPtr.asFunction< void Function( @@ -693,7 +696,8 @@ class NativeLibrary { int, int, ffi.Pointer>, - ffi.Pointer, + ffi.Pointer>, + int, double)>(); void play_animation(