diff --git a/ios/Classes/filament/FilamentMethodCallHandler.mm b/ios/Classes/filament/FilamentMethodCallHandler.mm index 9e468ee9..d1e03e92 100644 --- a/ios/Classes/filament/FilamentMethodCallHandler.mm +++ b/ios/Classes/filament/FilamentMethodCallHandler.mm @@ -63,30 +63,58 @@ static void* freeResourceGlobal(void* mem, size_t size, void* misc) { if(!_viewer) return; _viewer->manipulator->grabBegin([call.arguments[0] intValue], [call.arguments[1] intValue], true); + result(@"OK"); } else if([@"panUpdate" isEqualToString:call.method]) { if(!_viewer) return; _viewer->manipulator->grabUpdate([call.arguments[0] intValue], [call.arguments[1] intValue]); + result(@"OK"); } else if([@"panEnd" isEqualToString:call.method]) { if(!_viewer) return; _viewer->manipulator->grabEnd(); + result(@"OK"); } else if([@"rotateStart" isEqualToString:call.method]) { if(!_viewer) return; _viewer->manipulator->grabBegin([call.arguments[0] intValue], [call.arguments[1] intValue], false); + result(@"OK"); } else if([@"rotateUpdate" isEqualToString:call.method]) { if(!_viewer) return; _viewer->manipulator->grabUpdate([call.arguments[0] intValue], [call.arguments[1] intValue]); + result(@"OK"); } else if([@"rotateEnd" isEqualToString:call.method]) { if(!_viewer) return; _viewer->manipulator->grabEnd(); + result(@"OK"); } else if([@"releaseSourceAssets" isEqualToString:call.method]) { _viewer->releaseSourceAssets(); + result(@"OK"); + } else if([@"animateWeights" isEqualToString:call.method]) { + NSArray* frameData = call.arguments[0]; + NSNumber* numWeights = call.arguments[1]; + NSNumber* frameRate = call.arguments[2]; + + float* framesArr = (float*)malloc([frameData count] *sizeof(float)); + for(int i =0 ; i < [frameData count]; i++) { + *(framesArr+i) = [[frameData objectAtIndex:i] floatValue]; + } + _viewer->animateWeights((float*)framesArr, [numWeights intValue], [frameData count], [frameRate floatValue]); } else if([@"createMorpher" isEqualToString:call.method]) { - _viewer->createMorpher([call.arguments[0] UTF8String], [call.arguments[1] intValue]); + const char* meshName = [call.arguments[0] UTF8String]; + NSArray* primitiveIndices = call.arguments[1]; + int* primitiveIndicesArr = (int*)malloc([primitiveIndices count] *sizeof(int)); + for(int i =0 ; i < [primitiveIndices count]; i++) { + primitiveIndicesArr[i] = [[primitiveIndices objectAtIndex:i] intValue]; + } + _viewer->createMorpher(meshName, primitiveIndicesArr, [primitiveIndices count]); + free(primitiveIndicesArr); + result(@"OK"); + } else if([@"playAnimation" isEqualToString:call.method]) { + _viewer->playAnimation([call.arguments intValue]); + result(@"OK"); } else if([@"getTargetNames" isEqualToString:call.method]) { mimetic::StringList list = _viewer->getTargetNames([call.arguments UTF8String]); NSMutableArray* asArray = [NSMutableArray arrayWithCapacity:list.count]; @@ -97,19 +125,22 @@ static void* freeResourceGlobal(void* mem, size_t size, void* misc) { } else if([@"applyWeights" isEqualToString:call.method]) { if(!_viewer) return; - NSArray* nWeights = call.arguments[0]; + NSArray* nWeights = call.arguments; int count = [nWeights count]; float weights[count]; for(int i=0; i < count; i++) { weights[i] = [nWeights[i] floatValue]; } - _viewer->morphHelper->applyWeights(weights, count); + _viewer->applyWeights(weights, count); + result(@"OK"); } else if([@"zoom" isEqualToString:call.method]) { if(!_viewer) return; _viewer->manipulator->scroll(0.0f, 0.0f, [call.arguments floatValue]); + result(@"OK"); } else { + result(FlutterMethodNotImplemented); } } diff --git a/ios/src/FilamentViewer.cpp b/ios/src/FilamentViewer.cpp index c51183ae..f3290c58 100644 --- a/ios/src/FilamentViewer.cpp +++ b/ios/src/FilamentViewer.cpp @@ -43,10 +43,14 @@ #include #include +#include "math.h" +#include "upcast.h" + +#include +#include +#include #include #include -#include -#include #include @@ -65,6 +69,8 @@ namespace filament { namespace gltfio { MaterialProvider* createGPUMorphShaderLoader(const void* data, uint64_t size, Engine* engine); + void decomposeMatrix(const filament::math::mat4f& mat, filament::math::float3* translation, + filament::math::quatf* rotation, filament::math::float3* scale); } namespace mimetic { @@ -76,13 +82,41 @@ const float kAperture = 16.0f; const float kShutterSpeed = 1.0f / 125.0f; const float kSensitivity = 100.0f; +filament::math::mat4f composeMatrix(const filament::math::float3& translation, + const filament::math::quatf& rotation, const filament::math::float3& scale) { + float tx = translation[0]; + float ty = translation[1]; + float tz = translation[2]; + float qx = rotation[0]; + float qy = rotation[1]; + float qz = rotation[2]; + float qw = rotation[3]; + float sx = scale[0]; + float sy = scale[1]; + float sz = scale[2]; + return filament::math::mat4f( + (1 - 2 * qy*qy - 2 * qz*qz) * sx, + (2 * qx*qy + 2 * qz*qw) * sx, + (2 * qx*qz - 2 * qy*qw) * sx, + 0.f, + (2 * qx*qy - 2 * qz*qw) * sy, + (1 - 2 * qx*qx - 2 * qz*qz) * sy, + (2 * qy*qz + 2 * qx*qw) * sy, + 0.f, + (2 * qx*qz + 2 * qy*qw) * sz, + (2 * qy*qz - 2 * qx*qw) * sz, + (1 - 2 * qx*qx - 2 * qy*qy) * sz, + 0.f, tx, ty, tz, 1.f); +} + FilamentViewer::FilamentViewer( void* layer, const char* shaderPath, LoadResource loadResource, FreeResource freeResource) : _layer(layer), - _loadResource(loadResource), - _freeResource(freeResource) { + _loadResource(loadResource), + _freeResource(freeResource), + materialProviderResources(nullptr, 0) { _engine = Engine::create(Engine::Backend::OPENGL); _renderer = _engine->createRenderer(); @@ -99,9 +133,9 @@ FilamentViewer::FilamentViewer( _swapChain = _engine->createSwapChain(_layer); if(shaderPath) { - ResourceBuffer rb = _loadResource(shaderPath); - _materialProvider = createGPUMorphShaderLoader(rb.data, rb.size, _engine); - // _freeResource((void*)rb.data, rb.size, nullptr); <- TODO this is being freed too early, need to pass to callback? + materialProviderResources = _loadResource(shaderPath); + _materialProvider = createGPUMorphShaderLoader(materialProviderResources.data, materialProviderResources.size, _engine); + // _freeResource((void*)rb.data, rb.size, nullptr); <- TODO this is being freed too early, need to pass to callback? } else { _materialProvider = createUbershaderLoader(_engine); } @@ -147,7 +181,52 @@ void FilamentViewer::loadResources(string relativeResourcePath) { }; void FilamentViewer::releaseSourceAssets() { - _asset->releaseSourceData(); + std::cout << "Releasing source data" << std::endl; + _asset->releaseSourceData(); + _freeResource((void*)materialProviderResources.data, materialProviderResources.size, nullptr); +} + +void FilamentViewer::animateWeightsInternal(float* data, int numWeights, int length, float frameRate) { + int frameIndex = 0; + int numFrames = length / numWeights; + float frameLength = 1000 / frameRate; + + applyWeights(data, numWeights); + auto animationStartTime = std::chrono::high_resolution_clock::now(); + while(frameIndex < numFrames) { + duration dur = std::chrono::high_resolution_clock::now() - animationStartTime; + int msElapsed = dur.count(); + if(msElapsed > frameLength) { + std::cout << "frame" << frameIndex << std::endl; + frameIndex++; + applyWeights(data + (frameIndex * numWeights), numWeights); + animationStartTime = std::chrono::high_resolution_clock::now(); + } + } +} + + + +void FilamentViewer::animateWeights(float* data, int numWeights, int length, float frameRate) { + int numFrames = length / numWeights; + float frameLength = 1000 / frameRate; + + thread* t = new thread( +[=](){ + int frameIndex = 0; + + applyWeights(data, numWeights); + auto animationStartTime = std::chrono::high_resolution_clock::now(); + while(frameIndex < numFrames) { + duration dur = std::chrono::high_resolution_clock::now() - animationStartTime; + int msElapsed = dur.count(); + if(msElapsed > frameLength) { + frameIndex++; + applyWeights(data + (frameIndex * numWeights), numWeights); + animationStartTime = std::chrono::high_resolution_clock::now(); + } + } + }); } void FilamentViewer::loadGltf(const char* const uri, const char* const relativeResourcePath) { @@ -195,8 +274,40 @@ StringList FilamentViewer::getTargetNames(const char* meshName) { return StringList(nullptr, 0); } -void FilamentViewer::createMorpher(const char* meshName, int primitiveIndex) { - morphHelper = new gltfio::GPUMorphHelper((FFilamentAsset*)_asset, meshName, primitiveIndex); +void FilamentViewer::createMorpher(const char* meshName, int* primitives, int numPrimitives) { + morphHelper = new gltfio::GPUMorphHelper((FFilamentAsset*)_asset, meshName, primitives, numPrimitives); +// morphHelper = new gltfio::CPUMorpher(((FFilamentAsset)*_asset, (FFilamentInstance*)_asset)); +} + +void FilamentViewer::applyWeights(float* weights, int count) { + morphHelper->applyWeights(weights, count); +} + +void FilamentViewer::animateBones() { + Entity entity = _asset->getFirstEntityByName("CC_Base_JawRoot"); + if(!entity) { + return; + } + + TransformManager& transformManager = _engine->getTransformManager(); + + TransformManager::Instance node = transformManager.getInstance( entity); + + mat4f xform = transformManager.getTransform(node); + float3 scale; + quatf rotation; + float3 translation; + decomposeMatrix(xform, &translation, &rotation, &scale); + +// const quatf srcQuat { weights[0] * 0.9238,0,weights[0] * 0.3826, 0 }; +// float3 { scale[0] * (1.0f - weights[0]), scale[1] * (1.0f - weights[1]), scale[2] * (1.0f - weights[2]) } +// xform = composeMatrix(translation + float3 { weights[0], weights[1], weights[2] }, rotation, scale ); + transformManager.setTransform(node, xform); + +} + +void FilamentViewer::playAnimation(int index) { + _activeAnimation = index; } @@ -261,7 +372,7 @@ void FilamentViewer::cleanup() { }; void FilamentViewer::render() { - if (!_view || !_mainCamera || !manipulator) { + if (!_view || !_mainCamera || !manipulator || !_animator) { return; } // Extract the camera basis from the helper and push it to the Filament camera. @@ -271,12 +382,12 @@ void FilamentViewer::render() { _mainCamera->lookAt(eye, target, upward); if(_animator) { - typedef std::chrono::duration duration; + duration dur = std::chrono::high_resolution_clock::now() - startTime; - //if (_animator->getAnimationCount() > 0) { - ///_animator->applyAnimation(0, dur.count() / 1000); - //} - //_animator->updateBoneMatrices(); + if (_activeAnimation >= 0 && _animator->getAnimationCount() > 0) { + _animator->applyAnimation(_activeAnimation, dur.count() / 1000); + _animator->updateBoneMatrices(); + } } // Render the scene, unless the renderer wants to skip the frame. diff --git a/ios/src/FilamentViewer.hpp b/ios/src/FilamentViewer.hpp index 51bd26fd..0ca9e193 100644 --- a/ios/src/FilamentViewer.hpp +++ b/ios/src/FilamentViewer.hpp @@ -50,10 +50,19 @@ namespace mimetic { struct ResourceBuffer { ResourceBuffer(const void* data, const uint32_t size) : data(data), size(size) {}; + + ResourceBuffer& operator=(ResourceBuffer other) + { + data = other.data; + size = other.size; + return *this; + } const void* data; - const uint64_t size; + uint64_t size; }; + typedef std::chrono::duration duration; + using LoadResource = std::function; using FreeResource = std::function; @@ -65,23 +74,31 @@ namespace mimetic { void loadSkybox(const char* const skyboxUri, const char* const iblUri); void updateViewportAndCameraProjection(int height, int width, float scaleFactor); void render(); - void createMorpher(const char* meshName, int primitiveIndex); + void createMorpher(const char* meshName, int* primitives, int numPrimitives); void releaseSourceAssets(); StringList getTargetNames(const char* meshName); Manipulator* manipulator; - GPUMorphHelper* morphHelper; + void applyWeights(float* weights, int count); + void animateWeights(float* data, int numWeights, int length, float frameRate); + void animateBones(); + void playAnimation(int index); private: + void loadResources(std::string relativeResourcePath); void transformToUnitCube(); void cleanup(); + void animateWeightsInternal(float* data, int numWeights, int length, float frameRate); void* _layer; - LoadResource _loadResource; FreeResource _freeResource; + + ResourceBuffer materialProviderResources; std::chrono::high_resolution_clock::time_point startTime; + + int _activeAnimation = -1; Scene* _scene; View* _view; @@ -113,6 +130,8 @@ namespace mimetic { float _cameraFocalLength = 0.0f; + GPUMorphHelper* morphHelper; + }; } diff --git a/ios/src/morph/CPUMorpher.cpp b/ios/src/morph/CPUMorpher.cpp new file mode 100644 index 00000000..ec38a581 --- /dev/null +++ b/ios/src/morph/CPUMorpher.cpp @@ -0,0 +1,310 @@ +// /* +// * Copyright (C) 2021 The Android Open Source Project +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// #include "CPUMorpher.h" + +// #include +// #include +// #include +// #include + +// #include "GltfHelpers.h" + +// using namespace filament; +// using namespace filament::math; +// using namespace utils; +// using namespace gltfio; + +// namespace agltfio { + +// static const auto FREE_CALLBACK = [](void* mem, size_t, void*) { free(mem); }; + +// CPUMorpher::CPUMorpher(FFilamentAsset* asset, FFilamentInstance* instance) : mAsset(asset) { +// NodeMap& sourceNodes = asset->isInstanced() ? asset->mInstances[0]->nodeMap : asset->mNodeMap; + +// int i = 0; +// for (auto pair : sourceNodes) { +// cgltf_node const* node = pair.first; +// cgltf_mesh const* mesh = node->mesh; +// if (mesh) { +// cgltf_primitive const* prims = mesh->primitives; +// for (cgltf_size pi = 0, count = mesh->primitives_count; pi < count; ++pi) { +// if (mesh->primitives[pi].targets_count > 0) { +// addPrimitive(mesh, pi, &mMorphTable[pair.second]); +// } +// } +// } +// } +// } + +// CPUMorpher::~CPUMorpher() { +// auto engine = mAsset->mEngine; +// for (auto& entry : mMorphTable) { +// for (auto& prim : entry.second.primitives) { +// if (prim.morphBuffer1) engine->destroy(prim.morphBuffer1); +// if (prim.morphBuffer2) engine->destroy(prim.morphBuffer2); +// } +// } +// } + +// void CPUMorpher::applyWeights(Entity entity, float const* weights, size_t count) noexcept { +// auto& engine = *mAsset->mEngine; +// auto renderableManager = &engine.getRenderableManager(); +// auto renderable = renderableManager->getInstance(entity); + +// for (auto& prim : mMorphTable[entity].primitives) { +// if (prim.targets.size() < count) { +// continue; +// } + +// size_t size = prim.floatsCount * sizeof(float); +// float* data = (float*) malloc(size); +// memset(data, 0, size); + +// for (size_t index = 0; index < count; index ++) { +// const float w = weights[index]; +// if (w < 0.0001f) continue; + +// const GltfTarget& target = prim.targets[index]; +// cgltf_size dim = cgltf_num_components(target.type); + +// for (size_t i = 0; i < target.indices.size(); ++i) { +// uint16_t index = target.indices[i]; +// for (size_t j = 0; j < dim; ++j) { +// data[index * dim + j] += w * target.values[i * dim + j]; +// } +// } +// } + +// VertexBuffer* vb = prim.vertexBuffer; +// if (!prim.morphBuffer1) { +// prim.morphBuffer1 = BufferObject::Builder().size(size).build(engine); +// } +// if (!prim.morphBuffer2 && count >= 2) { +// // This is for dealing with a bug in filament shaders where empty normals are not +// // handled correctly. +// // +// // filament shaders deal with tangent frame quaternions instead of normal vectors. +// // But in case of missing inputs, we get a invalid quaternion (0, 0, 0, 0) instead of +// // a identity quaterion. This leads to the normals being morphed even no inputs are +// // given. +// // +// // To fix this, we put a empty morph target at slot 2 and give it a weight of -1. +// // This won't affect the vertex positions but will cancel out the normal values. +// // +// // Note that for this to work, at least two morph targets are required. +// float* data2 = (float*) malloc(size); +// memset(data2, 0, size); +// VertexBuffer::BufferDescriptor bd2(data2, size, FREE_CALLBACK); +// prim.morphBuffer2 = BufferObject::Builder().size(size).build(engine); +// prim.morphBuffer2->setBuffer(engine, std::move(bd2)); +// vb->setBufferObjectAt(engine, prim.baseSlot+1, prim.morphBuffer2); +// } +// VertexBuffer::BufferDescriptor bd(data, size, FREE_CALLBACK); +// prim.morphBuffer1->setBuffer(engine, std::move(bd)); +// vb->setBufferObjectAt(engine, prim.baseSlot, prim.morphBuffer1); +// } +// renderableManager->setMorphWeights(renderable, {1, -1, 0, 0}); +// } + +// // // This method copies various morphing-related data from the FilamentAsset MeshCache primitive +// // // (which lives in transient memory) into the MorphHelper primitive (which will stay resident). +// // void MorphHelper::addPrimitive(cgltf_mesh const* mesh, int primitiveIndex, TableEntry* entry) { +// // auto& engine = *mAsset->mEngine; +// // const cgltf_primitive& prim = mesh->primitives[primitiveIndex]; +// // const auto& gltfioPrim = mAsset->mMeshCache.at(mesh)[primitiveIndex]; +// // VertexBuffer* vertexBuffer = gltfioPrim.vertices; + +// // entry->primitives.push_back({ vertexBuffer }); +// // auto& morphHelperPrim = entry->primitives.back(); + +// // for (int i = 0; i < 4; i++) { +// // morphHelperPrim.positions[i] = gltfioPrim.morphPositions[i]; +// // morphHelperPrim.tangents[i] = gltfioPrim.morphTangents[i]; +// // } + +// // const cgltf_accessor* previous = nullptr; +// // for (int targetIndex = 0; targetIndex < prim.targets_count; targetIndex++) { +// // const cgltf_morph_target& morphTarget = prim.targets[targetIndex]; +// // for (cgltf_size aindex = 0; aindex < morphTarget.attributes_count; aindex++) { +// // const cgltf_attribute& attribute = morphTarget.attributes[aindex]; +// // const cgltf_accessor* accessor = attribute.data; +// // const cgltf_attribute_type atype = attribute.type; +// // if (atype == cgltf_attribute_type_tangent) { +// // continue; +// // } +// // if (atype == cgltf_attribute_type_normal) { + +// // // TODO: use JobSystem for this, like what we do for non-morph tangents. +// // TangentsJob job; +// // TangentsJob::Params params = { .in = { &prim, targetIndex } }; +// // TangentsJob::run(¶ms); + +// // if (params.out.results) { +// // const size_t size = params.out.vertexCount * sizeof(short4); +// // BufferObject* bufferObject = BufferObject::Builder().size(size).build(engine); +// // VertexBuffer::BufferDescriptor bd(params.out.results, size, FREE_CALLBACK); +// // bufferObject->setBuffer(engine, std::move(bd)); +// // params.out.results = nullptr; +// // morphHelperPrim.targets.push_back({bufferObject, targetIndex, atype}); +// // } +// // continue; +// // } +// // if (atype == cgltf_attribute_type_position) { +// // // All position attributes must have the same data type. +// // assert_invariant(!previous || previous->component_type == accessor->component_type); +// // assert_invariant(!previous || previous->type == accessor->type); +// // previous = accessor; + +// // // This should always be non-null, but don't crash if the glTF is malformed. +// // if (accessor->buffer_view) { +// // auto bufferData = (const uint8_t*) accessor->buffer_view->buffer->data; +// // assert_invariant(bufferData); +// // const uint8_t* data = computeBindingOffset(accessor) + bufferData; +// // const uint32_t size = computeBindingSize(accessor); + +// // // This creates a copy because we don't know when the user will free the cgltf +// // // source data. For non-morphed vertex buffers, we use a sharing mechanism to +// // // prevent copies, but here we just want to keep it as simple as possible. +// // uint8_t* clone = (uint8_t*) malloc(size); +// // memcpy(clone, data, size); + +// // BufferObject* bufferObject = BufferObject::Builder().size(size).build(engine); +// // VertexBuffer::BufferDescriptor bd(clone, size, FREE_CALLBACK); +// // bufferObject->setBuffer(engine, std::move(bd)); +// // morphHelperPrim.targets.push_back({bufferObject, targetIndex, atype}); +// // } +// // } +// // } +// // } +// // } + +// void CPUMorpher::addPrimitive(cgltf_mesh const* mesh, int primitiveIndex, TableEntry* entry) { + +// auto& engine = *mAsset->mEngine; +// const cgltf_primitive& cgltf_prim = mesh->primitives[primitiveIndex]; + +// int posIndex = findPositionAttribute(cgltf_prim); +// if (posIndex < 0) return; + +// VertexBuffer* vertexBuffer = mAsset->mMeshCache.at(mesh)[primitiveIndex].vertices; +// int slot = determineBaseSlot(cgltf_prim); +// entry->primitives.push_back({ vertexBuffer, slot }); + +// auto& primitive = entry->primitives.back(); + +// cgltf_attribute& positionAttribute = cgltf_prim.attributes[posIndex]; +// size_t dim = cgltf_num_components(positionAttribute.data->type); +// primitive.floatsCount = positionAttribute.data->count * dim; + +// std::vector& targets = primitive.targets; + +// for (int targetIndex = 0; targetIndex < cgltf_prim.targets_count; targetIndex++) { +// const cgltf_morph_target& morphTarget = cgltf_prim.targets[targetIndex]; +// for (cgltf_size aindex = 0; aindex < morphTarget.attributes_count; aindex++) { +// const cgltf_attribute& attribute = morphTarget.attributes[aindex]; +// const cgltf_accessor* accessor = attribute.data; +// const cgltf_attribute_type atype = attribute.type; + +// // only works for morphing of positions for now +// if (atype == cgltf_attribute_type_position || atype == cgltf_attribute_type_normal) { +// targets.push_back({targetIndex, atype, accessor->type}); +// cgltf_size floats_per_element = cgltf_num_components(accessor->type); + +// targets.back().indices.resize(accessor->count); +// targets.back().values.resize(accessor->count * floats_per_element); +// cgltf_size unpacked = cgltf_accessor_unpack_floats(accessor, targets.back().values.data(), accessor->count * floats_per_element); +// for(int i = 0; i < accessor->count; i++) { +// targets.back().indices[i] = i; +// /* for(int j = 0; j < floats_per_element; j++) { +// size_t offset = (i * floats_per_element) + j; +// cgltf_element_read_float +// float cgv = cgltf_accessor_unpack_floats(accessor->buffer_view + offset, accessor->component_type, accessor->normalized); +// if(cgv > 0) { +// size_t foo = 1; +// targets.back().values[offset] = cgv; +// } +// targets.back().values[offset] = cgv; +// } */ +// } +// //cgltf_size unpacked = cgltf_accessor_unpack_floats(accessor, targets.back().values.data(), floats_per_element); +// } +// } +// } +// } + +// // trying to find the slot for the vertex position +// int CPUMorpher::determineBaseSlot(const cgltf_primitive& prim) const { +// int slot = 0; +// bool hasNormals = false; +// for (cgltf_size aindex = 0; aindex < prim.attributes_count; aindex++) { +// const cgltf_attribute& attribute = prim.attributes[aindex]; +// const int index = attribute.index; +// const cgltf_attribute_type atype = attribute.type; +// const cgltf_accessor* accessor = attribute.data; +// if (atype == cgltf_attribute_type_tangent) { +// continue; +// } +// if (atype == cgltf_attribute_type_normal) { +// slot++; +// hasNormals = true; +// continue; +// } + +// // Filament supports two set of texcoords. But whether or not UV1 is actually used also +// // depends on the material. +// // Note this is a very specific fix that might not work in all cases. +// if (atype == cgltf_attribute_type_texcoord && index > 0) { +// bool hasUV1 = false; +// cgltf_material* mat = prim.material; +// if (mat->has_pbr_metallic_roughness) { +// if (mat->pbr_metallic_roughness.base_color_texture.texcoord != 0) { +// hasUV1 = true; +// } +// if (mat->pbr_metallic_roughness.metallic_roughness_texture.texcoord != 0) { +// hasUV1 = true; +// } +// } +// if (mat->normal_texture.texcoord != 0) hasUV1 = true; +// if (mat->emissive_texture.texcoord != 0) hasUV1 = true; +// if (mat->occlusion_texture.texcoord != 0) hasUV1 = true; + +// if (hasUV1) slot++; +// continue; +// } + +// slot++; +// } + +// // If the model has lighting but not normals, then a slot is used for generated flat normals. +// if (prim.material && !prim.material->unlit && !hasNormals) { +// slot++; +// } + +// return slot; +// }; + +// int CPUMorpher::findPositionAttribute(const cgltf_primitive& prim) const { +// for (cgltf_size aindex = 0; aindex < prim.attributes_count; aindex++) { +// if (prim.attributes[aindex].type == cgltf_attribute_type_position) { +// return aindex; +// } +// } + +// return -1; +// }; + +// } // namespace gltfio diff --git a/ios/src/morph/CPUMorpher.h b/ios/src/morph/CPUMorpher.h new file mode 100644 index 00000000..48d7c939 --- /dev/null +++ b/ios/src/morph/CPUMorpher.h @@ -0,0 +1,91 @@ +// /* +// * Copyright (C) 2021 The Android Open Source Project +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// #ifndef GLTFIO_CPUMORPHER_H +// #define GLTFIO_CPUMORPHER_H + +// #include "Morpher.h" + +// #include "FFilamentAsset.h" +// #include "FFilamentInstance.h" + +// #include + +// #include + +// #include + +// struct cgltf_node; +// struct cgltf_mesh; +// struct cgltf_primitive; +// struct cgltf_attribute; + +// using namespace gltfio; + + +// namespace agltfio { + +// /** +// * Helper for supporting more than 4 active morph targets. +// * +// * All morph values are calculated on CPU and collected into a single target, which will be +// * uploaded with weight of 1. This is effectively doing the morphing on CPU. +// * +// * Obviously this is slower than the stock morpher as it needs to upload buffer every frame. +// * So beware of the performance penalty. +// */ +// class CPUMorpher : public Morpher { +// public: +// using Entity = utils::Entity; +// CPUMorpher(FFilamentAsset* asset, FFilamentInstance* instance); +// ~CPUMorpher(); + +// void applyWeights(Entity targetEntity, float const* weights, size_t count) noexcept; + +// private: +// struct GltfTarget { +// int morphTargetIndex; +// cgltf_attribute_type attribute_type; +// cgltf_type type; +// std::vector indices; +// std::vector values; +// }; + +// struct GltfPrimitive { +// filament::VertexBuffer* vertexBuffer; +// int baseSlot; +// size_t floatsCount; +// filament::BufferObject* morphBuffer1 = nullptr; +// filament::BufferObject* morphBuffer2 = nullptr; +// std::vector targets; +// }; + +// struct TableEntry { +// std::vector primitives; +// }; + +// void addPrimitive(cgltf_mesh const* mesh, int primitiveIndex, TableEntry* entry); +// int determineBaseSlot(const cgltf_primitive& prim) const; +// int findPositionAttribute(const cgltf_primitive& prim) const; + +// std::vector mPartiallySortedWeights; +// tsl::robin_map mMorphTable; +// const FFilamentAsset* mAsset; +// }; + +// } // namespace gltfio + +// #endif // GLTFIO_CPUMORPHER_H diff --git a/ios/src/morph/GPUMorphHelper.cpp b/ios/src/morph/GPUMorphHelper.cpp index c68ceff2..efef037d 100644 --- a/ios/src/morph/GPUMorphHelper.cpp +++ b/ios/src/morph/GPUMorphHelper.cpp @@ -41,17 +41,14 @@ namespace gltfio { uint32_t computeBindingSize(const cgltf_accessor *accessor); - uint32_t computeBindingSize(const cgltf_accessor *accessor); - uint32_t computeBindingOffset(const cgltf_accessor *accessor); static const auto FREE_CALLBACK = [](void *mem, size_t s, void *) { free(mem); }; - GPUMorphHelper::GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int primitiveIndex) : mAsset(asset), mPrimitiveIndex(primitiveIndex) { + GPUMorphHelper::GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int* primitiveIndices, int numPrimitives) : mAsset(asset) { - cgltf_size num_primitives = 0; NodeMap &sourceNodes = asset->isInstanced() ? asset->mInstances[0]->nodeMap : asset->mNodeMap; @@ -60,11 +57,12 @@ namespace gltfio { cgltf_mesh const *mesh = node->mesh; if (mesh) { - std::cout << "Mesh " << mesh->name <name << " with " << mesh->weights_count << " weights " << std::endl; if(strcmp(meshName, mesh->name) == 0) { targetMesh = mesh; - num_primitives = mesh->primitives_count; - addPrimitive(mesh); + std::cout << "Adding primitive to mesh with " << mesh->primitives_count << " primitives." << std::endl; + for(int i = 0; i < numPrimitives; i++) + addPrimitive(mesh, primitiveIndices[i]); } } } @@ -77,6 +75,8 @@ namespace gltfio { } createTextures(); + + applyWeights(targetMesh->weights, targetMesh->weights_count); } @@ -91,99 +91,110 @@ namespace gltfio { auto materialInstances = mAsset->getMaterialInstances(); auto &engine = *(mAsset->mEngine); - auto& prim = animatedPrimitive; + for(auto& prim : animatablePrimitives) { + + // for a single morph target, each vertex will be assigned 2 pixels, corresponding to a position vec3 and a normal vec3 + // these two vectors will be laid out adjacent in memory + // the total texture "width" is the total number of these pixels + // morph targets are then assigned to the depth channel + auto textureWidth = prim->numVertices * numAttributes; - // for a single morph target, each vertex will be assigned 2 pixels, corresponding to a position vec3 and a normal vec3 - // these two vectors will be laid out adjacent in memory - // the total texture "width" is the total number of these pixels - // morph targets are then assigned to the depth channel - auto textureWidth = prim->numVertices * 2; - - // the total size of the texture in bytes - // equal to (numVertices * numAttributes * vectorSize (3) * sizeof(float) * numMorphTargets) - auto textureSize = textureWidth * 3 * sizeof(float) * prim->numTargets; - auto textureBuffer = (float *const) malloc(textureSize); - - if(!textureBuffer) { - std::cout << "Error allocating texture buffer" << std::endl; - exit(-1); - } - - uint32_t offset = 0; - - // assume the primitive morph target source buffer is laid out like: - // |target0_v0_pos * 3|target0_v0_norm * 3|target0_v1_pos * 3|target0_v1_norm * 3|...|target1_v0_pos * 3|target1_v0_norm * 3|target1_v1_pos * 3|target1_v1_norm * 3|... - // where: - // - target0/target1/etc is the first/second/etc morph target - // - v0/v1/etc is the first/second/etc vertex - // - pos/norm are each 3-float vectors - for (auto &target : prim->targets) { - if(target.type == cgltf_attribute_type_position - || target.type == cgltf_attribute_type_normal - ) { - memcpy(textureBuffer+offset, target.bufferObject, target.bufferSize); - offset += int(target.bufferSize / sizeof(float)); - } - } - - Texture* texture = Texture::Builder() - .width(textureWidth) // - .height(1) - .depth(prim->numTargets) - .sampler(Texture::Sampler::SAMPLER_2D_ARRAY) - .format(Texture::InternalFormat::RGB32F) - .levels(0x01) - .build(engine); + // the total size of the texture in bytes + // equal to (numVertices * numAttributes * vectorSize (3) * sizeof(float) * numMorphTargets) + auto textureSize = textureWidth * 3 * sizeof(float) * prim->numTargets; + auto textureBuffer = (float *const) malloc(textureSize); - prim->texture = texture; //std::unique_ptr(texture); - - Texture::PixelBufferDescriptor descriptor( - textureBuffer, - textureSize, - Texture::Format::RGB, - Texture::Type::FLOAT, - FREE_CALLBACK, - nullptr); - prim->texture->setImage(engine, 0, 0,0, 0, textureWidth, 1, prim->numTargets, std::move(descriptor)); - for(int i = 0; i < mAsset->getMaterialInstanceCount(); i++) { - const char* name = materialInstances[i]->getName(); - if(strcmp(name, prim->materialName) == 0) { - std::cout << "Found material instance for primitive under name : " << name << std::endl; - prim->materialInstance = materialInstances[i]; //std::unique_ptr(materialInstances[i]); - break; - } - } - - if(!prim->materialInstance) { - exit(-1); - } + if(!textureBuffer) { + std::cout << "Error allocating texture buffer" << std::endl; + exit(-1); + } + + memset(textureBuffer, 0, textureSize); + + uint32_t offset = 0; + + // assume the primitive morph target source buffer is laid out like: + // |target0_v0_pos * 3|target0_v0_norm * 3|target0_v1_pos * 3|target0_v1_norm * 3|...|target1_v0_pos * 3|target1_v0_norm * 3|target1_v1_pos * 3|target1_v1_norm * 3|... + // where: + // - target0/target1/etc is the first/second/etc morph target + // - v0/v1/etc is the first/second/etc vertex + // - pos/norm are each 3-float vectors + for (auto &target : prim->targets) { + if(target.type == cgltf_attribute_type_position + || (numAttributes > 1 && target.type == cgltf_attribute_type_normal) + ) { - prim->materialInstance->setParameter("dimensions", filament::math::int3 { prim->numVertices * 2, numAttributes, prim->numTargets }); - prim->materialInstance->setParameter("morphTargets", prim->texture, TextureSampler()); - float weights[prim->numTargets]; - memset(weights, 0, prim->numTargets * sizeof(float)); - prim->materialInstance->setParameter("morphTargetWeights", weights, prim->numTargets); + memcpy(textureBuffer+offset, target.bufferObject, target.bufferSize); + offset += int(target.bufferSize / sizeof(float)); + } + } + + Texture* texture = Texture::Builder() + .width(textureWidth) // + .height(1) + .depth(prim->numTargets) + .sampler(Texture::Sampler::SAMPLER_2D_ARRAY) + .format(Texture::InternalFormat::RGB32F) + .levels(0x01) + .build(engine); + + prim->texture = texture; //std::unique_ptr(texture); + + Texture::PixelBufferDescriptor descriptor( + textureBuffer, + textureSize, + Texture::Format::RGB, + Texture::Type::FLOAT, + FREE_CALLBACK, + nullptr); + prim->texture->setImage(engine, 0, 0,0, 0, textureWidth, 1, prim->numTargets, std::move(descriptor)); + + for(int i = 0; i < mAsset->getMaterialInstanceCount(); i++) { + const char* name = materialInstances[i]->getName(); + if(strcmp(name, prim->materialName) == 0) { + std::cout << "Found material instance for primitive under name : " << name << std::endl; + prim->materialInstance = materialInstances[i]; //std::unique_ptr(materialInstances[i]); + break; + } + } + + if(!prim->materialInstance) { + exit(-1); + } + + float dimensions[] = { (float(prim->numVertices) * float(numAttributes)), float(numAttributes), float(prim->numTargets) }; + + prim->materialInstance->setParameter("dimensions", dimensions, 3); + // TextureSampler sampler(filament::backend::SamplerMagFilter::NEAREST, filament::TextureSampler::WrapMode::REPEAT); + + prim->materialInstance->setParameter("morphTargets", prim->texture, TextureSampler()); + float weights[prim->numTargets]; + memset(weights, 0, prim->numTargets * sizeof(float)); + prim->materialInstance->setParameter("morphTargetWeights", weights, prim->numTargets); + } } void GPUMorphHelper::applyWeights(float const *weights, size_t count) noexcept { - std::cout << "Applying " << count << " weights " << std::endl; - animatedPrimitive->materialInstance->setParameter("morphTargetWeights", weights, count); + for(auto& prim : animatablePrimitives) { + prim->materialInstance->setParameter("morphTargetWeights", weights, count); + } } void - GPUMorphHelper::addPrimitive(cgltf_mesh const *mesh) { + GPUMorphHelper::addPrimitive(cgltf_mesh const *mesh, int primitiveIndex) { auto &engine = *mAsset->mEngine; - const cgltf_primitive &prim = mesh->primitives[mPrimitiveIndex]; + const cgltf_primitive &prim = mesh->primitives[primitiveIndex]; - const auto &gltfioPrim = mAsset->mMeshCache.at(mesh)[mPrimitiveIndex]; + const auto &gltfioPrim = mAsset->mMeshCache.at(mesh)[primitiveIndex]; VertexBuffer *vertexBuffer = gltfioPrim.vertices; - - animatedPrimitive = std::make_unique(GltfPrimitive{vertexBuffer}); + std::unique_ptr animatedPrimitive = std::make_unique(GltfPrimitive{vertexBuffer}); animatedPrimitive->materialName = prim.material->name; animatedPrimitive->numTargets = prim.targets_count; animatedPrimitive->numVertices = vertexBuffer->getVertexCount(); + + std::cout << "Found " << animatedPrimitive->numVertices << " vertices in primitive" << std::endl; cgltf_size maxIndex = 0; for(int i = 0; i < prim.indices->count; i++) { maxIndex = std::max(cgltf_accessor_read_index(prim.indices, i), maxIndex); @@ -202,14 +213,10 @@ namespace gltfio { continue; } if ( - atype == cgltf_attribute_type_normal || - atype == cgltf_attribute_type_position + atype == cgltf_attribute_type_position || (numAttributes > 1 && atype == cgltf_attribute_type_normal) ) { - // - // the texture needs to be sized according to the total number of vertices in the mesh - // this is identified by the highest vertex index of all primitives in the mesh - + // All position & normal attributes must have the same data type. assert_invariant( !previous || previous->component_type == accessor->component_type); @@ -222,10 +229,12 @@ namespace gltfio { assert_invariant(bufferData); const uint8_t *data = computeBindingOffset(accessor) + bufferData; const uint32_t size = computeBindingSize(accessor); + animatedPrimitive->targets.push_back({data, size, targetIndex, atype}); } } } } + animatablePrimitives.push_back(std::move(animatedPrimitive)); } } diff --git a/ios/src/morph/GPUMorphHelper.h b/ios/src/morph/GPUMorphHelper.h index 4b67c4f7..065d4062 100644 --- a/ios/src/morph/GPUMorphHelper.h +++ b/ios/src/morph/GPUMorphHelper.h @@ -42,14 +42,14 @@ namespace gltfio { public: using Entity = utils::Entity; - GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int primitiveIndex); + GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int* primitives, int numPrimitives); ~GPUMorphHelper(); void applyWeights(float const *weights, size_t count) noexcept; private: - int mPrimitiveIndex; + struct GltfTarget { const void *bufferObject; uint32_t bufferSize; @@ -67,17 +67,15 @@ namespace gltfio { MaterialInstance* materialInstance; }; - int numAttributes = 2; // position & normal + int numAttributes = 1; // just position for now - normals not working with indexing inside shader? byte offset seems not calculated correctly - uint32_t* indicesBuffer = nullptr; - - void addPrimitive(cgltf_mesh const *mesh); + void addPrimitive(cgltf_mesh const *mesh, int primitiveIndex); void createTextures(); cgltf_mesh const* targetMesh; FFilamentAsset *mAsset; - std::unique_ptr animatedPrimitive; + std::vector> animatablePrimitives; }; } diff --git a/lib/filament_controller.dart b/lib/filament_controller.dart index 6197046b..cf8666ea 100644 --- a/lib/filament_controller.dart +++ b/lib/filament_controller.dart @@ -14,12 +14,14 @@ abstract class FilamentController { Future rotateStart(double x, double y); Future rotateUpdate(double x, double y); Future rotateEnd(); - Future applyWeights(List weights, int primitiveIndex); + Future applyWeights(List weights); Future> getTargetNames(String meshName); + Future releaseSourceAssets(); + Future playAnimation(int index); - void animate( - List> weights, int primitiveIndex, double frameRate); - Future createMorpher(String meshName, int primitiveIndex); + // Weights is expected to be a contiguous sequence of floats of size W*F, where W is the number of weights and F is the number of frames + void animate(List weights, int numWeights, double frameRate); + Future createMorpher(String meshName, List primitives); Future zoom(double z); } @@ -90,8 +92,8 @@ class MimeticFilamentController extends FilamentController { await _channel.invokeMethod("rotateEnd"); } - Future applyWeights(List weights, int primitiveIndex) async { - await _channel.invokeMethod("applyWeights", [weights, primitiveIndex]); + Future applyWeights(List weights) async { + await _channel.invokeMethod("applyWeights", weights); } Future> getTargetNames(String meshName) async { @@ -100,18 +102,8 @@ class MimeticFilamentController extends FilamentController { return result; } - void animate( - List> weights, int primitiveIndex, double frameRate) async { - final msPerFrame = 1000 ~/ frameRate; - int i = 0; - - Timer.periodic(Duration(milliseconds: msPerFrame), (t) async { - _channel.invokeMethod("applyWeights", [weights[i], primitiveIndex]); - i++; - if (i >= weights.length) { - t.cancel(); - } - }); + void animate(List weights, int numWeights, double frameRate) async { + _channel.invokeMethod("animateWeights", [weights, numWeights, frameRate]); } Future releaseSourceAssets() async { @@ -122,7 +114,11 @@ class MimeticFilamentController extends FilamentController { await _channel.invokeMethod("zoom", z); } - Future createMorpher(String meshName, int primitiveIndex) async { - await _channel.invokeMethod("createMorpher", [meshName, primitiveIndex]); + Future createMorpher(String meshName, List primitives) async { + await _channel.invokeMethod("createMorpher", [meshName, primitives]); + } + + Future playAnimation(int index) async { + await _channel.invokeMethod("playAnimation", index); } }