maobo/cup_edit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add support for multiple primitives

Browse Source
This commit is contained in:
Nick Fisher
2021-09-24 11:47:50 +08:00
parent 14c8f431a2
commit 1361fc7d41
8 changed files with 704 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
ios/Classes/filament/FilamentMethodCallHandler.mm
View File

@@ -63,30 +63,58 @@ static void* freeResourceGlobal(void* mem, size_t size, void* misc) {
if(!_viewer) if(!_viewer)
return; return;
_viewer->manipulator->grabBegin([call.arguments[0] intValue], [call.arguments[1] intValue], true); _viewer->manipulator->grabBegin([call.arguments[0] intValue], [call.arguments[1] intValue], true);
result(@"OK");
} else if([@"panUpdate" isEqualToString:call.method]) { } else if([@"panUpdate" isEqualToString:call.method]) {
if(!_viewer) if(!_viewer)
return; return;
_viewer->manipulator->grabUpdate([call.arguments[0] intValue], [call.arguments[1] intValue]); _viewer->manipulator->grabUpdate([call.arguments[0] intValue], [call.arguments[1] intValue]);
result(@"OK");
} else if([@"panEnd" isEqualToString:call.method]) { } else if([@"panEnd" isEqualToString:call.method]) {
if(!_viewer) if(!_viewer)
return; return;
_viewer->manipulator->grabEnd(); _viewer->manipulator->grabEnd();
result(@"OK");
} else if([@"rotateStart" isEqualToString:call.method]) { } else if([@"rotateStart" isEqualToString:call.method]) {
if(!_viewer) if(!_viewer)
return; return;
_viewer->manipulator->grabBegin([call.arguments[0] intValue], [call.arguments[1] intValue], false); _viewer->manipulator->grabBegin([call.arguments[0] intValue], [call.arguments[1] intValue], false);
result(@"OK");
} else if([@"rotateUpdate" isEqualToString:call.method]) { } else if([@"rotateUpdate" isEqualToString:call.method]) {
if(!_viewer) if(!_viewer)
return; return;
_viewer->manipulator->grabUpdate([call.arguments[0] intValue], [call.arguments[1] intValue]); _viewer->manipulator->grabUpdate([call.arguments[0] intValue], [call.arguments[1] intValue]);
result(@"OK");
} else if([@"rotateEnd" isEqualToString:call.method]) { } else if([@"rotateEnd" isEqualToString:call.method]) {
if(!_viewer) if(!_viewer)
return; return;
_viewer->manipulator->grabEnd(); _viewer->manipulator->grabEnd();
result(@"OK");
} else if([@"releaseSourceAssets" isEqualToString:call.method]) { } else if([@"releaseSourceAssets" isEqualToString:call.method]) {
_viewer->releaseSourceAssets(); _viewer->releaseSourceAssets();
result(@"OK");
} else if([@"animateWeights" isEqualToString:call.method]) {
NSArray* frameData = call.arguments[0];
NSNumber* numWeights = call.arguments[1];
NSNumber* frameRate = call.arguments[2];
float* framesArr = (float*)malloc([frameData count] *sizeof(float));
for(int i =0 ; i < [frameData count]; i++) {
*(framesArr+i) = [[frameData objectAtIndex:i] floatValue];
}
_viewer->animateWeights((float*)framesArr, [numWeights intValue], [frameData count], [frameRate floatValue]);
} else if([@"createMorpher" isEqualToString:call.method]) { } else if([@"createMorpher" isEqualToString:call.method]) {
_viewer->createMorpher([call.arguments[0] UTF8String], [call.arguments[1] intValue]); const char* meshName = [call.arguments[0] UTF8String];
NSArray* primitiveIndices = call.arguments[1];
int* primitiveIndicesArr = (int*)malloc([primitiveIndices count] *sizeof(int));
for(int i =0 ; i < [primitiveIndices count]; i++) {
primitiveIndicesArr[i] = [[primitiveIndices objectAtIndex:i] intValue];
}
_viewer->createMorpher(meshName, primitiveIndicesArr, [primitiveIndices count]);
free(primitiveIndicesArr);
result(@"OK");
} else if([@"playAnimation" isEqualToString:call.method]) {
_viewer->playAnimation([call.arguments intValue]);
result(@"OK");
} else if([@"getTargetNames" isEqualToString:call.method]) { } else if([@"getTargetNames" isEqualToString:call.method]) {
mimetic::StringList list = _viewer->getTargetNames([call.arguments UTF8String]); mimetic::StringList list = _viewer->getTargetNames([call.arguments UTF8String]);
NSMutableArray* asArray = [NSMutableArray arrayWithCapacity:list.count]; NSMutableArray* asArray = [NSMutableArray arrayWithCapacity:list.count];
@@ -97,19 +125,22 @@ static void* freeResourceGlobal(void* mem, size_t size, void* misc) {
} else if([@"applyWeights" isEqualToString:call.method]) { } else if([@"applyWeights" isEqualToString:call.method]) {
if(!_viewer) if(!_viewer)
return; return;
NSArray* nWeights = call.arguments[0]; NSArray* nWeights = call.arguments;
int count = [nWeights count]; int count = [nWeights count];
float weights[count]; float weights[count];
for(int i=0; i < count; i++) { for(int i=0; i < count; i++) {
weights[i] = [nWeights[i] floatValue]; weights[i] = [nWeights[i] floatValue];
} }
_viewer->morphHelper->applyWeights(weights, count); _viewer->applyWeights(weights, count);
result(@"OK");
} else if([@"zoom" isEqualToString:call.method]) { } else if([@"zoom" isEqualToString:call.method]) {
if(!_viewer) if(!_viewer)
return; return;
_viewer->manipulator->scroll(0.0f, 0.0f, [call.arguments floatValue]); _viewer->manipulator->scroll(0.0f, 0.0f, [call.arguments floatValue]);
result(@"OK");
} else { } else {
result(FlutterMethodNotImplemented); result(FlutterMethodNotImplemented);
} }
} }

137
ios/src/FilamentViewer.cpp
View File

@@ -43,10 +43,14 @@
#include <utils/NameComponentManager.h> #include <utils/NameComponentManager.h>
#include <utils/JobSystem.h> #include <utils/JobSystem.h>
#include "math.h"
#include "upcast.h"
#include <math/mat4.h>
#include <math/quat.h>
#include <math/scalar.h>
#include <math/vec3.h> #include <math/vec3.h>
#include <math/vec4.h> #include <math/vec4.h>
#include <math/mat3.h>
#include <math/norm.h>
#include <image/KtxUtility.h> #include <image/KtxUtility.h>
@@ -65,6 +69,8 @@ namespace filament {
namespace gltfio { namespace gltfio {
MaterialProvider* createGPUMorphShaderLoader(const void* data, uint64_t size, Engine* engine); MaterialProvider* createGPUMorphShaderLoader(const void* data, uint64_t size, Engine* engine);
void decomposeMatrix(const filament::math::mat4f& mat, filament::math::float3* translation,
filament::math::quatf* rotation, filament::math::float3* scale);
} }
namespace mimetic { namespace mimetic {
@@ -76,13 +82,41 @@ const float kAperture = 16.0f;
const float kShutterSpeed = 1.0f / 125.0f; const float kShutterSpeed = 1.0f / 125.0f;
const float kSensitivity = 100.0f; const float kSensitivity = 100.0f;
filament::math::mat4f composeMatrix(const filament::math::float3& translation,
const filament::math::quatf& rotation, const filament::math::float3& scale) {
float tx = translation[0];
float ty = translation[1];
float tz = translation[2];
float qx = rotation[0];
float qy = rotation[1];
float qz = rotation[2];
float qw = rotation[3];
float sx = scale[0];
float sy = scale[1];
float sz = scale[2];
return filament::math::mat4f(
(1 - 2 * qy*qy - 2 * qz*qz) * sx,
(2 * qx*qy + 2 * qz*qw) * sx,
(2 * qx*qz - 2 * qy*qw) * sx,
0.f,
(2 * qx*qy - 2 * qz*qw) * sy,
(1 - 2 * qx*qx - 2 * qz*qz) * sy,
(2 * qy*qz + 2 * qx*qw) * sy,
0.f,
(2 * qx*qz + 2 * qy*qw) * sz,
(2 * qy*qz - 2 * qx*qw) * sz,
(1 - 2 * qx*qx - 2 * qy*qy) * sz,
0.f, tx, ty, tz, 1.f);
}
FilamentViewer::FilamentViewer( FilamentViewer::FilamentViewer(
void* layer, void* layer,
const char* shaderPath, const char* shaderPath,
LoadResource loadResource, LoadResource loadResource,
FreeResource freeResource) : _layer(layer), FreeResource freeResource) : _layer(layer),
_loadResource(loadResource), _loadResource(loadResource),
_freeResource(freeResource) { _freeResource(freeResource),
materialProviderResources(nullptr, 0) {
_engine = Engine::create(Engine::Backend::OPENGL); _engine = Engine::create(Engine::Backend::OPENGL);
_renderer = _engine->createRenderer(); _renderer = _engine->createRenderer();
@@ -99,8 +133,8 @@ FilamentViewer::FilamentViewer(
_swapChain = _engine->createSwapChain(_layer); _swapChain = _engine->createSwapChain(_layer);
if(shaderPath) { if(shaderPath) {
ResourceBuffer rb = _loadResource(shaderPath); materialProviderResources = _loadResource(shaderPath);
_materialProvider = createGPUMorphShaderLoader(rb.data, rb.size, _engine); _materialProvider = createGPUMorphShaderLoader(materialProviderResources.data, materialProviderResources.size, _engine);
// _freeResource((void*)rb.data, rb.size, nullptr); <- TODO this is being freed too early, need to pass to callback? // _freeResource((void*)rb.data, rb.size, nullptr); <- TODO this is being freed too early, need to pass to callback?
} else { } else {
_materialProvider = createUbershaderLoader(_engine); _materialProvider = createUbershaderLoader(_engine);
@@ -147,7 +181,52 @@ void FilamentViewer::loadResources(string relativeResourcePath) {
}; };
void FilamentViewer::releaseSourceAssets() { void FilamentViewer::releaseSourceAssets() {
std::cout << "Releasing source data" << std::endl;
_asset->releaseSourceData(); _asset->releaseSourceData();
_freeResource((void*)materialProviderResources.data, materialProviderResources.size, nullptr);
}
void FilamentViewer::animateWeightsInternal(float* data, int numWeights, int length, float frameRate) {
int frameIndex = 0;
int numFrames = length / numWeights;
float frameLength = 1000 / frameRate;
applyWeights(data, numWeights);
auto animationStartTime = std::chrono::high_resolution_clock::now();
while(frameIndex < numFrames) {
duration dur = std::chrono::high_resolution_clock::now() - animationStartTime;
int msElapsed = dur.count();
if(msElapsed > frameLength) {
std::cout << "frame" << frameIndex << std::endl;
frameIndex++;
applyWeights(data + (frameIndex * numWeights), numWeights);
animationStartTime = std::chrono::high_resolution_clock::now();
}
}
}
void FilamentViewer::animateWeights(float* data, int numWeights, int length, float frameRate) {
int numFrames = length / numWeights;
float frameLength = 1000 / frameRate;
thread* t = new thread(
[=](){
int frameIndex = 0;
applyWeights(data, numWeights);
auto animationStartTime = std::chrono::high_resolution_clock::now();
while(frameIndex < numFrames) {
duration dur = std::chrono::high_resolution_clock::now() - animationStartTime;
int msElapsed = dur.count();
if(msElapsed > frameLength) {
frameIndex++;
applyWeights(data + (frameIndex * numWeights), numWeights);
animationStartTime = std::chrono::high_resolution_clock::now();
}
}
});
} }
void FilamentViewer::loadGltf(const char* const uri, const char* const relativeResourcePath) { void FilamentViewer::loadGltf(const char* const uri, const char* const relativeResourcePath) {
@@ -195,8 +274,40 @@ StringList FilamentViewer::getTargetNames(const char* meshName) {
return StringList(nullptr, 0); return StringList(nullptr, 0);
} }
void FilamentViewer::createMorpher(const char* meshName, int primitiveIndex) { void FilamentViewer::createMorpher(const char* meshName, int* primitives, int numPrimitives) {
morphHelper = new gltfio::GPUMorphHelper((FFilamentAsset*)_asset, meshName, primitiveIndex); morphHelper = new gltfio::GPUMorphHelper((FFilamentAsset*)_asset, meshName, primitives, numPrimitives);
// morphHelper = new gltfio::CPUMorpher(((FFilamentAsset)*_asset, (FFilamentInstance*)_asset));
}
void FilamentViewer::applyWeights(float* weights, int count) {
morphHelper->applyWeights(weights, count);
}
void FilamentViewer::animateBones() {
Entity entity = _asset->getFirstEntityByName("CC_Base_JawRoot");
if(!entity) {
return;
}
TransformManager& transformManager = _engine->getTransformManager();
TransformManager::Instance node = transformManager.getInstance( entity);
mat4f xform = transformManager.getTransform(node);
float3 scale;
quatf rotation;
float3 translation;
decomposeMatrix(xform, &translation, &rotation, &scale);
// const quatf srcQuat { weights[0] * 0.9238,0,weights[0] * 0.3826, 0 };
// float3 { scale[0] * (1.0f - weights[0]), scale[1] * (1.0f - weights[1]), scale[2] * (1.0f - weights[2]) }
// xform = composeMatrix(translation + float3 { weights[0], weights[1], weights[2] }, rotation, scale );
transformManager.setTransform(node, xform);
}
void FilamentViewer::playAnimation(int index) {
_activeAnimation = index;
} }
@@ -261,7 +372,7 @@ void FilamentViewer::cleanup() {
}; };
void FilamentViewer::render() { void FilamentViewer::render() {
if (!_view || !_mainCamera || !manipulator) { if (!_view || !_mainCamera || !manipulator || !_animator) {
return; return;
} }
// Extract the camera basis from the helper and push it to the Filament camera. // Extract the camera basis from the helper and push it to the Filament camera.
@@ -271,12 +382,12 @@ void FilamentViewer::render() {
_mainCamera->lookAt(eye, target, upward); _mainCamera->lookAt(eye, target, upward);
if(_animator) { if(_animator) {
typedef std::chrono::duration<float, std::milli> duration;
duration dur = std::chrono::high_resolution_clock::now() - startTime; duration dur = std::chrono::high_resolution_clock::now() - startTime;
//if (_animator->getAnimationCount() > 0) { if (_activeAnimation >= 0 && _animator->getAnimationCount() > 0) {
///_animator->applyAnimation(0, dur.count() / 1000); _animator->applyAnimation(_activeAnimation, dur.count() / 1000);
//} _animator->updateBoneMatrices();
//_animator->updateBoneMatrices(); }
} }
// Render the scene, unless the renderer wants to skip the frame. // Render the scene, unless the renderer wants to skip the frame.

27
ios/src/FilamentViewer.hpp
View File

@@ -50,10 +50,19 @@ namespace mimetic {
struct ResourceBuffer { struct ResourceBuffer {
ResourceBuffer(const void* data, const uint32_t size) : data(data), size(size) {}; ResourceBuffer(const void* data, const uint32_t size) : data(data), size(size) {};
ResourceBuffer& operator=(ResourceBuffer other)
{
data = other.data;
size = other.size;
return *this;
}
const void* data; const void* data;
const uint64_t size; uint64_t size;
}; };
typedef std::chrono::duration<float, std::milli> duration;
using LoadResource = std::function<ResourceBuffer(const char* uri)>; using LoadResource = std::function<ResourceBuffer(const char* uri)>;
using FreeResource = std::function<void * (void *mem, size_t s, void *)>; using FreeResource = std::function<void * (void *mem, size_t s, void *)>;
@@ -65,24 +74,32 @@ namespace mimetic {
void loadSkybox(const char* const skyboxUri, const char* const iblUri); void loadSkybox(const char* const skyboxUri, const char* const iblUri);
void updateViewportAndCameraProjection(int height, int width, float scaleFactor); void updateViewportAndCameraProjection(int height, int width, float scaleFactor);
void render(); void render();
void createMorpher(const char* meshName, int primitiveIndex); void createMorpher(const char* meshName, int* primitives, int numPrimitives);
void releaseSourceAssets(); void releaseSourceAssets();
StringList getTargetNames(const char* meshName); StringList getTargetNames(const char* meshName);
Manipulator<float>* manipulator; Manipulator<float>* manipulator;
GPUMorphHelper* morphHelper; void applyWeights(float* weights, int count);
void animateWeights(float* data, int numWeights, int length, float frameRate);
void animateBones();
void playAnimation(int index);
private: private:
void loadResources(std::string relativeResourcePath); void loadResources(std::string relativeResourcePath);
void transformToUnitCube(); void transformToUnitCube();
void cleanup(); void cleanup();
void animateWeightsInternal(float* data, int numWeights, int length, float frameRate);
void* _layer; void* _layer;
LoadResource _loadResource; LoadResource _loadResource;
FreeResource _freeResource; FreeResource _freeResource;
ResourceBuffer materialProviderResources;
std::chrono::high_resolution_clock::time_point startTime; std::chrono::high_resolution_clock::time_point startTime;
int _activeAnimation = -1;
Scene* _scene; Scene* _scene;
View* _view; View* _view;
Engine* _engine; Engine* _engine;
@@ -113,6 +130,8 @@ namespace mimetic {
float _cameraFocalLength = 0.0f; float _cameraFocalLength = 0.0f;
GPUMorphHelper* morphHelper;
}; };
} }

310
ios/src/morph/CPUMorpher.cpp Normal file
View File

@@ -0,0 +1,310 @@
// /*
// * Copyright (C) 2021 The Android Open Source Project
// *
// * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// * You may obtain a copy of the License at
// *
// * http://www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */
// #include "CPUMorpher.h"
// #include <filament/BufferObject.h>
// #include <filament/RenderableManager.h>
// #include <filament/VertexBuffer.h>
// #include <utils/Log.h>
// #include "GltfHelpers.h"
// using namespace filament;
// using namespace filament::math;
// using namespace utils;
// using namespace gltfio;
// namespace agltfio {
// static const auto FREE_CALLBACK = [](void* mem, size_t, void*) { free(mem); };
// CPUMorpher::CPUMorpher(FFilamentAsset* asset, FFilamentInstance* instance) : mAsset(asset) {
// NodeMap& sourceNodes = asset->isInstanced() ? asset->mInstances[0]->nodeMap : asset->mNodeMap;
// int i = 0;
// for (auto pair : sourceNodes) {
// cgltf_node const* node = pair.first;
// cgltf_mesh const* mesh = node->mesh;
// if (mesh) {
// cgltf_primitive const* prims = mesh->primitives;
// for (cgltf_size pi = 0, count = mesh->primitives_count; pi < count; ++pi) {
// if (mesh->primitives[pi].targets_count > 0) {
// addPrimitive(mesh, pi, &mMorphTable[pair.second]);
// }
// }
// }
// }
// }
// CPUMorpher::~CPUMorpher() {
// auto engine = mAsset->mEngine;
// for (auto& entry : mMorphTable) {
// for (auto& prim : entry.second.primitives) {
// if (prim.morphBuffer1) engine->destroy(prim.morphBuffer1);
// if (prim.morphBuffer2) engine->destroy(prim.morphBuffer2);
// }
// }
// }
// void CPUMorpher::applyWeights(Entity entity, float const* weights, size_t count) noexcept {
// auto& engine = *mAsset->mEngine;
// auto renderableManager = &engine.getRenderableManager();
// auto renderable = renderableManager->getInstance(entity);
// for (auto& prim : mMorphTable[entity].primitives) {
// if (prim.targets.size() < count) {
// continue;
// }
// size_t size = prim.floatsCount * sizeof(float);
// float* data = (float*) malloc(size);
// memset(data, 0, size);
// for (size_t index = 0; index < count; index ++) {
// const float w = weights[index];
// if (w < 0.0001f) continue;
// const GltfTarget& target = prim.targets[index];
// cgltf_size dim = cgltf_num_components(target.type);
// for (size_t i = 0; i < target.indices.size(); ++i) {
// uint16_t index = target.indices[i];
// for (size_t j = 0; j < dim; ++j) {
// data[index * dim + j] += w * target.values[i * dim + j];
// }
// }
// }
// VertexBuffer* vb = prim.vertexBuffer;
// if (!prim.morphBuffer1) {
// prim.morphBuffer1 = BufferObject::Builder().size(size).build(engine);
// }
// if (!prim.morphBuffer2 && count >= 2) {
// // This is for dealing with a bug in filament shaders where empty normals are not
// // handled correctly.
// //
// // filament shaders deal with tangent frame quaternions instead of normal vectors.
// // But in case of missing inputs, we get a invalid quaternion (0, 0, 0, 0) instead of
// // a identity quaterion. This leads to the normals being morphed even no inputs are
// // given.
// //
// // To fix this, we put a empty morph target at slot 2 and give it a weight of -1.
// // This won't affect the vertex positions but will cancel out the normal values.
// //
// // Note that for this to work, at least two morph targets are required.
// float* data2 = (float*) malloc(size);
// memset(data2, 0, size);
// VertexBuffer::BufferDescriptor bd2(data2, size, FREE_CALLBACK);
// prim.morphBuffer2 = BufferObject::Builder().size(size).build(engine);
// prim.morphBuffer2->setBuffer(engine, std::move(bd2));
// vb->setBufferObjectAt(engine, prim.baseSlot+1, prim.morphBuffer2);
// }
// VertexBuffer::BufferDescriptor bd(data, size, FREE_CALLBACK);
// prim.morphBuffer1->setBuffer(engine, std::move(bd));
// vb->setBufferObjectAt(engine, prim.baseSlot, prim.morphBuffer1);
// }
// renderableManager->setMorphWeights(renderable, {1, -1, 0, 0});
// }
// // // This method copies various morphing-related data from the FilamentAsset MeshCache primitive
// // // (which lives in transient memory) into the MorphHelper primitive (which will stay resident).
// // void MorphHelper::addPrimitive(cgltf_mesh const* mesh, int primitiveIndex, TableEntry* entry) {
// // auto& engine = *mAsset->mEngine;
// // const cgltf_primitive& prim = mesh->primitives[primitiveIndex];
// // const auto& gltfioPrim = mAsset->mMeshCache.at(mesh)[primitiveIndex];
// // VertexBuffer* vertexBuffer = gltfioPrim.vertices;
// // entry->primitives.push_back({ vertexBuffer });
// // auto& morphHelperPrim = entry->primitives.back();
// // for (int i = 0; i < 4; i++) {
// // morphHelperPrim.positions[i] = gltfioPrim.morphPositions[i];
// // morphHelperPrim.tangents[i] = gltfioPrim.morphTangents[i];
// // }
// // const cgltf_accessor* previous = nullptr;
// // for (int targetIndex = 0; targetIndex < prim.targets_count; targetIndex++) {
// // const cgltf_morph_target& morphTarget = prim.targets[targetIndex];
// // for (cgltf_size aindex = 0; aindex < morphTarget.attributes_count; aindex++) {
// // const cgltf_attribute& attribute = morphTarget.attributes[aindex];
// // const cgltf_accessor* accessor = attribute.data;
// // const cgltf_attribute_type atype = attribute.type;
// // if (atype == cgltf_attribute_type_tangent) {
// // continue;
// // }
// // if (atype == cgltf_attribute_type_normal) {
// // // TODO: use JobSystem for this, like what we do for non-morph tangents.
// // TangentsJob job;
// // TangentsJob::Params params = { .in = { &prim, targetIndex } };
// // TangentsJob::run(&params);
// // if (params.out.results) {
// // const size_t size = params.out.vertexCount * sizeof(short4);
// // BufferObject* bufferObject = BufferObject::Builder().size(size).build(engine);
// // VertexBuffer::BufferDescriptor bd(params.out.results, size, FREE_CALLBACK);
// // bufferObject->setBuffer(engine, std::move(bd));
// // params.out.results = nullptr;
// // morphHelperPrim.targets.push_back({bufferObject, targetIndex, atype});
// // }
// // continue;
// // }
// // if (atype == cgltf_attribute_type_position) {
// // // All position attributes must have the same data type.
// // assert_invariant(!previous || previous->component_type == accessor->component_type);
// // assert_invariant(!previous || previous->type == accessor->type);
// // previous = accessor;
// // // This should always be non-null, but don't crash if the glTF is malformed.
// // if (accessor->buffer_view) {
// // auto bufferData = (const uint8_t*) accessor->buffer_view->buffer->data;
// // assert_invariant(bufferData);
// // const uint8_t* data = computeBindingOffset(accessor) + bufferData;
// // const uint32_t size = computeBindingSize(accessor);
// // // This creates a copy because we don't know when the user will free the cgltf
// // // source data. For non-morphed vertex buffers, we use a sharing mechanism to
// // // prevent copies, but here we just want to keep it as simple as possible.
// // uint8_t* clone = (uint8_t*) malloc(size);
// // memcpy(clone, data, size);
// // BufferObject* bufferObject = BufferObject::Builder().size(size).build(engine);
// // VertexBuffer::BufferDescriptor bd(clone, size, FREE_CALLBACK);
// // bufferObject->setBuffer(engine, std::move(bd));
// // morphHelperPrim.targets.push_back({bufferObject, targetIndex, atype});
// // }
// // }
// // }
// // }
// // }
// void CPUMorpher::addPrimitive(cgltf_mesh const* mesh, int primitiveIndex, TableEntry* entry) {
// auto& engine = *mAsset->mEngine;
// const cgltf_primitive& cgltf_prim = mesh->primitives[primitiveIndex];
// int posIndex = findPositionAttribute(cgltf_prim);
// if (posIndex < 0) return;
// VertexBuffer* vertexBuffer = mAsset->mMeshCache.at(mesh)[primitiveIndex].vertices;
// int slot = determineBaseSlot(cgltf_prim);
// entry->primitives.push_back({ vertexBuffer, slot });
// auto& primitive = entry->primitives.back();
// cgltf_attribute& positionAttribute = cgltf_prim.attributes[posIndex];
// size_t dim = cgltf_num_components(positionAttribute.data->type);
// primitive.floatsCount = positionAttribute.data->count * dim;
// std::vector<GltfTarget>& targets = primitive.targets;
// for (int targetIndex = 0; targetIndex < cgltf_prim.targets_count; targetIndex++) {
// const cgltf_morph_target& morphTarget = cgltf_prim.targets[targetIndex];
// for (cgltf_size aindex = 0; aindex < morphTarget.attributes_count; aindex++) {
// const cgltf_attribute& attribute = morphTarget.attributes[aindex];
// const cgltf_accessor* accessor = attribute.data;
// const cgltf_attribute_type atype = attribute.type;
// // only works for morphing of positions for now
// if (atype == cgltf_attribute_type_position || atype == cgltf_attribute_type_normal) {
// targets.push_back({targetIndex, atype, accessor->type});
// cgltf_size floats_per_element = cgltf_num_components(accessor->type);
// targets.back().indices.resize(accessor->count);
// targets.back().values.resize(accessor->count * floats_per_element);
// cgltf_size unpacked = cgltf_accessor_unpack_floats(accessor, targets.back().values.data(), accessor->count * floats_per_element);
// for(int i = 0; i < accessor->count; i++) {
// targets.back().indices[i] = i;
// /* for(int j = 0; j < floats_per_element; j++) {
// size_t offset = (i * floats_per_element) + j;
// cgltf_element_read_float
// float cgv = cgltf_accessor_unpack_floats(accessor->buffer_view + offset, accessor->component_type, accessor->normalized);
// if(cgv > 0) {
// size_t foo = 1;
// targets.back().values[offset] = cgv;
// }
// targets.back().values[offset] = cgv;
// } */
// }
// //cgltf_size unpacked = cgltf_accessor_unpack_floats(accessor, targets.back().values.data(), floats_per_element);
// }
// }
// }
// }
// // trying to find the slot for the vertex position
// int CPUMorpher::determineBaseSlot(const cgltf_primitive& prim) const {
// int slot = 0;
// bool hasNormals = false;
// for (cgltf_size aindex = 0; aindex < prim.attributes_count; aindex++) {
// const cgltf_attribute& attribute = prim.attributes[aindex];
// const int index = attribute.index;
// const cgltf_attribute_type atype = attribute.type;
// const cgltf_accessor* accessor = attribute.data;
// if (atype == cgltf_attribute_type_tangent) {
// continue;
// }
// if (atype == cgltf_attribute_type_normal) {
// slot++;
// hasNormals = true;
// continue;
// }
// // Filament supports two set of texcoords. But whether or not UV1 is actually used also
// // depends on the material.
// // Note this is a very specific fix that might not work in all cases.
// if (atype == cgltf_attribute_type_texcoord && index > 0) {
// bool hasUV1 = false;
// cgltf_material* mat = prim.material;
// if (mat->has_pbr_metallic_roughness) {
// if (mat->pbr_metallic_roughness.base_color_texture.texcoord != 0) {
// hasUV1 = true;
// }
// if (mat->pbr_metallic_roughness.metallic_roughness_texture.texcoord != 0) {
// hasUV1 = true;
// }
// }
// if (mat->normal_texture.texcoord != 0) hasUV1 = true;
// if (mat->emissive_texture.texcoord != 0) hasUV1 = true;
// if (mat->occlusion_texture.texcoord != 0) hasUV1 = true;
// if (hasUV1) slot++;
// continue;
// }
// slot++;
// }
// // If the model has lighting but not normals, then a slot is used for generated flat normals.
// if (prim.material && !prim.material->unlit && !hasNormals) {
// slot++;
// }
// return slot;
// };
// int CPUMorpher::findPositionAttribute(const cgltf_primitive& prim) const {
// for (cgltf_size aindex = 0; aindex < prim.attributes_count; aindex++) {
// if (prim.attributes[aindex].type == cgltf_attribute_type_position) {
// return aindex;
// }
// }
// return -1;
// };
// } // namespace gltfio

91
ios/src/morph/CPUMorpher.h Normal file
View File

@@ -0,0 +1,91 @@
// /*
// * Copyright (C) 2021 The Android Open Source Project
// *
// * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// * You may obtain a copy of the License at
// *
// * http://www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */
// #ifndef GLTFIO_CPUMORPHER_H
// #define GLTFIO_CPUMORPHER_H
// #include "Morpher.h"
// #include "FFilamentAsset.h"
// #include "FFilamentInstance.h"
// #include <math/vec4.h>
// #include <tsl/robin_map.h>
// #include <vector>
// struct cgltf_node;
// struct cgltf_mesh;
// struct cgltf_primitive;
// struct cgltf_attribute;
// using namespace gltfio;
// namespace agltfio {
// /**
// * Helper for supporting more than 4 active morph targets.
// *
// * All morph values are calculated on CPU and collected into a single target, which will be
// * uploaded with weight of 1. This is effectively doing the morphing on CPU.
// *
// * Obviously this is slower than the stock morpher as it needs to upload buffer every frame.
// * So beware of the performance penalty.
// */
// class CPUMorpher : public Morpher {
// public:
// using Entity = utils::Entity;
// CPUMorpher(FFilamentAsset* asset, FFilamentInstance* instance);
// ~CPUMorpher();
// void applyWeights(Entity targetEntity, float const* weights, size_t count) noexcept;
// private:
// struct GltfTarget {
// int morphTargetIndex;
// cgltf_attribute_type attribute_type;
// cgltf_type type;
// std::vector<uint16_t> indices;
// std::vector<float> values;
// };
// struct GltfPrimitive {
// filament::VertexBuffer* vertexBuffer;
// int baseSlot;
// size_t floatsCount;
// filament::BufferObject* morphBuffer1 = nullptr;
// filament::BufferObject* morphBuffer2 = nullptr;
// std::vector<GltfTarget> targets;
// };
// struct TableEntry {
// std::vector<GltfPrimitive> primitives;
// };
// void addPrimitive(cgltf_mesh const* mesh, int primitiveIndex, TableEntry* entry);
// int determineBaseSlot(const cgltf_primitive& prim) const;
// int findPositionAttribute(const cgltf_primitive& prim) const;
// std::vector<float> mPartiallySortedWeights;
// tsl::robin_map<Entity, TableEntry> mMorphTable;
// const FFilamentAsset* mAsset;
// };
// } // namespace gltfio
// #endif // GLTFIO_CPUMORPHER_H

55
ios/src/morph/GPUMorphHelper.cpp
View File

@@ -41,17 +41,14 @@ namespace gltfio {
uint32_t computeBindingSize(const cgltf_accessor *accessor); uint32_t computeBindingSize(const cgltf_accessor *accessor);
uint32_t computeBindingSize(const cgltf_accessor *accessor);
uint32_t computeBindingOffset(const cgltf_accessor *accessor); uint32_t computeBindingOffset(const cgltf_accessor *accessor);
static const auto FREE_CALLBACK = [](void *mem, size_t s, void *) { static const auto FREE_CALLBACK = [](void *mem, size_t s, void *) {
free(mem); free(mem);
}; };
GPUMorphHelper::GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int primitiveIndex) : mAsset(asset), mPrimitiveIndex(primitiveIndex) { GPUMorphHelper::GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int* primitiveIndices, int numPrimitives) : mAsset(asset) {
cgltf_size num_primitives = 0;
NodeMap &sourceNodes = asset->isInstanced() ? asset->mInstances[0]->nodeMap NodeMap &sourceNodes = asset->isInstanced() ? asset->mInstances[0]->nodeMap
: asset->mNodeMap; : asset->mNodeMap;
@@ -60,11 +57,12 @@ namespace gltfio {
cgltf_mesh const *mesh = node->mesh; cgltf_mesh const *mesh = node->mesh;
if (mesh) { if (mesh) {
std::cout << "Mesh " << mesh->name <<std::endl; std::cout << "Mesh " << mesh->name << " with " << mesh->weights_count << " weights " << std::endl;
if(strcmp(meshName, mesh->name) == 0) { if(strcmp(meshName, mesh->name) == 0) {
targetMesh = mesh; targetMesh = mesh;
num_primitives = mesh->primitives_count; std::cout << "Adding primitive to mesh with " << mesh->primitives_count << " primitives." << std::endl;
addPrimitive(mesh); for(int i = 0; i < numPrimitives; i++)
addPrimitive(mesh, primitiveIndices[i]);
} }
} }
} }
@@ -78,6 +76,8 @@ namespace gltfio {
createTextures(); createTextures();
applyWeights(targetMesh->weights, targetMesh->weights_count);
} }
GPUMorphHelper::~GPUMorphHelper() { GPUMorphHelper::~GPUMorphHelper() {
@@ -91,24 +91,27 @@ namespace gltfio {
auto materialInstances = mAsset->getMaterialInstances(); auto materialInstances = mAsset->getMaterialInstances();
auto &engine = *(mAsset->mEngine); auto &engine = *(mAsset->mEngine);
auto& prim = animatedPrimitive; for(auto& prim : animatablePrimitives) {
// for a single morph target, each vertex will be assigned 2 pixels, corresponding to a position vec3 and a normal vec3 // for a single morph target, each vertex will be assigned 2 pixels, corresponding to a position vec3 and a normal vec3
// these two vectors will be laid out adjacent in memory // these two vectors will be laid out adjacent in memory
// the total texture "width" is the total number of these pixels // the total texture "width" is the total number of these pixels
// morph targets are then assigned to the depth channel // morph targets are then assigned to the depth channel
auto textureWidth = prim->numVertices * 2; auto textureWidth = prim->numVertices * numAttributes;
// the total size of the texture in bytes // the total size of the texture in bytes
// equal to (numVertices * numAttributes * vectorSize (3) * sizeof(float) * numMorphTargets) // equal to (numVertices * numAttributes * vectorSize (3) * sizeof(float) * numMorphTargets)
auto textureSize = textureWidth * 3 * sizeof(float) * prim->numTargets; auto textureSize = textureWidth * 3 * sizeof(float) * prim->numTargets;
auto textureBuffer = (float *const) malloc(textureSize); auto textureBuffer = (float *const) malloc(textureSize);
if(!textureBuffer) { if(!textureBuffer) {
std::cout << "Error allocating texture buffer" << std::endl; std::cout << "Error allocating texture buffer" << std::endl;
exit(-1); exit(-1);
} }
memset(textureBuffer, 0, textureSize);
uint32_t offset = 0; uint32_t offset = 0;
// assume the primitive morph target source buffer is laid out like: // assume the primitive morph target source buffer is laid out like:
@@ -119,8 +122,9 @@ namespace gltfio {
// - pos/norm are each 3-float vectors // - pos/norm are each 3-float vectors
for (auto &target : prim->targets) { for (auto &target : prim->targets) {
if(target.type == cgltf_attribute_type_position if(target.type == cgltf_attribute_type_position
|| target.type == cgltf_attribute_type_normal || (numAttributes > 1 && target.type == cgltf_attribute_type_normal)
) { ) {
memcpy(textureBuffer+offset, target.bufferObject, target.bufferSize); memcpy(textureBuffer+offset, target.bufferObject, target.bufferSize);
offset += int(target.bufferSize / sizeof(float)); offset += int(target.bufferSize / sizeof(float));
} }
@@ -159,31 +163,38 @@ namespace gltfio {
exit(-1); exit(-1);
} }
prim->materialInstance->setParameter("dimensions", filament::math::int3 { prim->numVertices * 2, numAttributes, prim->numTargets }); float dimensions[] = { (float(prim->numVertices) * float(numAttributes)), float(numAttributes), float(prim->numTargets) };
prim->materialInstance->setParameter("dimensions", dimensions, 3);
// TextureSampler sampler(filament::backend::SamplerMagFilter::NEAREST, filament::TextureSampler::WrapMode::REPEAT);
prim->materialInstance->setParameter("morphTargets", prim->texture, TextureSampler()); prim->materialInstance->setParameter("morphTargets", prim->texture, TextureSampler());
float weights[prim->numTargets]; float weights[prim->numTargets];
memset(weights, 0, prim->numTargets * sizeof(float)); memset(weights, 0, prim->numTargets * sizeof(float));
prim->materialInstance->setParameter("morphTargetWeights", weights, prim->numTargets); prim->materialInstance->setParameter("morphTargetWeights", weights, prim->numTargets);
} }
}
void GPUMorphHelper::applyWeights(float const *weights, size_t count) noexcept { void GPUMorphHelper::applyWeights(float const *weights, size_t count) noexcept {
std::cout << "Applying " << count << " weights " << std::endl; for(auto& prim : animatablePrimitives) {
animatedPrimitive->materialInstance->setParameter("morphTargetWeights", weights, count); prim->materialInstance->setParameter("morphTargetWeights", weights, count);
}
} }
void void
GPUMorphHelper::addPrimitive(cgltf_mesh const *mesh) { GPUMorphHelper::addPrimitive(cgltf_mesh const *mesh, int primitiveIndex) {
auto &engine = *mAsset->mEngine; auto &engine = *mAsset->mEngine;
const cgltf_primitive &prim = mesh->primitives[mPrimitiveIndex]; const cgltf_primitive &prim = mesh->primitives[primitiveIndex];
const auto &gltfioPrim = mAsset->mMeshCache.at(mesh)[mPrimitiveIndex]; const auto &gltfioPrim = mAsset->mMeshCache.at(mesh)[primitiveIndex];
VertexBuffer *vertexBuffer = gltfioPrim.vertices; VertexBuffer *vertexBuffer = gltfioPrim.vertices;
std::unique_ptr<GltfPrimitive> animatedPrimitive = std::make_unique<GltfPrimitive>(GltfPrimitive{vertexBuffer});
animatedPrimitive = std::make_unique<GltfPrimitive>(GltfPrimitive{vertexBuffer});
animatedPrimitive->materialName = prim.material->name; animatedPrimitive->materialName = prim.material->name;
animatedPrimitive->numTargets = prim.targets_count; animatedPrimitive->numTargets = prim.targets_count;
animatedPrimitive->numVertices = vertexBuffer->getVertexCount(); animatedPrimitive->numVertices = vertexBuffer->getVertexCount();
std::cout << "Found " << animatedPrimitive->numVertices << " vertices in primitive" << std::endl;
cgltf_size maxIndex = 0; cgltf_size maxIndex = 0;
for(int i = 0; i < prim.indices->count; i++) { for(int i = 0; i < prim.indices->count; i++) {
maxIndex = std::max(cgltf_accessor_read_index(prim.indices, i), maxIndex); maxIndex = std::max(cgltf_accessor_read_index(prim.indices, i), maxIndex);
@@ -202,13 +213,9 @@ namespace gltfio {
continue; continue;
} }
if ( if (
atype == cgltf_attribute_type_normal || atype == cgltf_attribute_type_position || (numAttributes > 1 && atype == cgltf_attribute_type_normal)
atype == cgltf_attribute_type_position
) { ) {
//
// the texture needs to be sized according to the total number of vertices in the mesh
// this is identified by the highest vertex index of all primitives in the mesh
// All position & normal attributes must have the same data type. // All position & normal attributes must have the same data type.
assert_invariant( assert_invariant(
@@ -222,10 +229,12 @@ namespace gltfio {
assert_invariant(bufferData); assert_invariant(bufferData);
const uint8_t *data = computeBindingOffset(accessor) + bufferData; const uint8_t *data = computeBindingOffset(accessor) + bufferData;
const uint32_t size = computeBindingSize(accessor); const uint32_t size = computeBindingSize(accessor);
animatedPrimitive->targets.push_back({data, size, targetIndex, atype}); animatedPrimitive->targets.push_back({data, size, targetIndex, atype});
} }
} }
} }
} }
animatablePrimitives.push_back(std::move(animatedPrimitive));
} }
} }

12
ios/src/morph/GPUMorphHelper.h
View File

@@ -42,14 +42,14 @@ namespace gltfio {
public: public:
using Entity = utils::Entity; using Entity = utils::Entity;
GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int primitiveIndex); GPUMorphHelper(FFilamentAsset *asset, const char* meshName, int* primitives, int numPrimitives);
~GPUMorphHelper(); ~GPUMorphHelper();
void applyWeights(float const *weights, size_t count) noexcept; void applyWeights(float const *weights, size_t count) noexcept;
private: private:
int mPrimitiveIndex;
struct GltfTarget { struct GltfTarget {
const void *bufferObject; const void *bufferObject;
uint32_t bufferSize; uint32_t bufferSize;
@@ -67,17 +67,15 @@ namespace gltfio {
MaterialInstance* materialInstance; MaterialInstance* materialInstance;
}; };
int numAttributes = 2; // position & normal int numAttributes = 1; // just position for now - normals not working with indexing inside shader? byte offset seems not calculated correctly
uint32_t* indicesBuffer = nullptr; void addPrimitive(cgltf_mesh const *mesh, int primitiveIndex);
void addPrimitive(cgltf_mesh const *mesh);
void createTextures(); void createTextures();
cgltf_mesh const* targetMesh; cgltf_mesh const* targetMesh;
FFilamentAsset *mAsset; FFilamentAsset *mAsset;
std::unique_ptr<GltfPrimitive> animatedPrimitive; std::vector<std::unique_ptr<GltfPrimitive>> animatablePrimitives;
}; };
} }

36
lib/filament_controller.dart
View File

@@ -14,12 +14,14 @@ abstract class FilamentController {
Future rotateStart(double x, double y); Future rotateStart(double x, double y);
Future rotateUpdate(double x, double y); Future rotateUpdate(double x, double y);
Future rotateEnd(); Future rotateEnd();
Future applyWeights(List<double> weights, int primitiveIndex); Future applyWeights(List<double> weights);
Future<List<String>> getTargetNames(String meshName); Future<List<String>> getTargetNames(String meshName);
Future releaseSourceAssets();
Future playAnimation(int index);
void animate( // Weights is expected to be a contiguous sequence of floats of size W*F, where W is the number of weights and F is the number of frames
List<List<double>> weights, int primitiveIndex, double frameRate); void animate(List<double> weights, int numWeights, double frameRate);
Future createMorpher(String meshName, int primitiveIndex); Future createMorpher(String meshName, List<int> primitives);
Future zoom(double z); Future zoom(double z);
} }
@@ -90,8 +92,8 @@ class MimeticFilamentController extends FilamentController {
await _channel.invokeMethod("rotateEnd"); await _channel.invokeMethod("rotateEnd");
} }
Future applyWeights(List<double> weights, int primitiveIndex) async { Future applyWeights(List<double> weights) async {
await _channel.invokeMethod("applyWeights", [weights, primitiveIndex]); await _channel.invokeMethod("applyWeights", weights);
} }
Future<List<String>> getTargetNames(String meshName) async { Future<List<String>> getTargetNames(String meshName) async {
@@ -100,18 +102,8 @@ class MimeticFilamentController extends FilamentController {
return result; return result;
} }
void animate( void animate(List<double> weights, int numWeights, double frameRate) async {
List<List<double>> weights, int primitiveIndex, double frameRate) async { _channel.invokeMethod("animateWeights", [weights, numWeights, frameRate]);
final msPerFrame = 1000 ~/ frameRate;
int i = 0;
Timer.periodic(Duration(milliseconds: msPerFrame), (t) async {
_channel.invokeMethod("applyWeights", [weights[i], primitiveIndex]);
i++;
if (i >= weights.length) {
t.cancel();
}
});
} }
Future releaseSourceAssets() async { Future releaseSourceAssets() async {
@@ -122,7 +114,11 @@ class MimeticFilamentController extends FilamentController {
await _channel.invokeMethod("zoom", z); await _channel.invokeMethod("zoom", z);
} }
Future createMorpher(String meshName, int primitiveIndex) async { Future createMorpher(String meshName, List<int> primitives) async {
await _channel.invokeMethod("createMorpher", [meshName, primitiveIndex]); await _channel.invokeMethod("createMorpher", [meshName, primitives]);
}
Future playAnimation(int index) async {
await _channel.invokeMethod("playAnimation", index);
} }
} }