add option to record to PNGs (inc background threadpool for encoding)

example/lib/main.dart

@@ -53,6 +53,7 @@ class ExampleWidgetState extends State<ExampleWidget> {
   // these are all the options that can be set via the menu
   // we store them here
   static bool rendering = false;
+  static bool recording = false;
   static int framerate = 60;
   static bool postProcessing = true;
   static bool frustumCulling = true;

example/lib/menus/rendering_submenu.dart

@@ -38,7 +38,7 @@ class _RenderingSubmenuState extends State<RenderingSubmenu> {
             widget.controller.setFrameRate(ExampleWidgetState.framerate);
           },
           child: Text(
-              "Toggle framerate (currently $ExampleWidgetState.framerate) "),
+              "Toggle framerate (currently ${ExampleWidgetState.framerate}) "),
         ),
         MenuItemButton(
           onPressed: () {
@@ -56,6 +56,14 @@ class _RenderingSubmenuState extends State<RenderingSubmenu> {
           child: Text(
               "${ExampleWidgetState.postProcessing ? "Disable" : "Enable"} postprocessing"),
         ),
+        MenuItemButton(
+          onPressed: () {
+            ExampleWidgetState.recording = !ExampleWidgetState.recording;
+            widget.controller.setRecording(ExampleWidgetState.recording);
+          },
+          child: Text(
+              "Turn recording ${ExampleWidgetState.recording ? "OFF" : "ON"}) "),
+        ),
       ],
       child: const Text("Rendering"),
     );

ios/include/FilamentViewer.hpp

@@ -33,6 +33,7 @@
 #include <chrono>
 
 #include "AssetManager.hpp"
+#include "ThreadPool.hpp"
 
 using namespace std;
 using namespace filament;
@@ -122,6 +123,8 @@ namespace polyvox
         void clearLights();
         void setPostProcessing(bool enabled);
 
+        void setRecording(bool recording);
+
 
         AssetManager *const getAssetManager()
         {
@@ -134,7 +137,7 @@ namespace polyvox
         Scene *_scene = nullptr;
         View *_view = nullptr;
         Engine *_engine = nullptr;
-
+        flutter_filament::ThreadPool* _tp = nullptr;
         Renderer *_renderer = nullptr;
         RenderTarget *_rt = nullptr;
         Texture *_rtColor = nullptr;
@@ -189,9 +192,13 @@ namespace polyvox
         void loadKtxTexture(string path, ResourceBuffer data);
         void loadPngTexture(string path, ResourceBuffer data);
         void loadTextureFromPath(string path);
-       
+        void savePng(void* data, size_t size);
 
+        int _frameNumber = 0;
+    
         uint32_t _lastFrameTimeInNanos;
+
+        bool _recording = false; 
     };
 
 }

ios/include/FlutterFilamentApi.h

@@ -169,6 +169,7 @@ extern "C"
 	FLUTTER_PLUGIN_EXPORT void pick(void *const viewer, int x, int y, EntityId *entityId);
 	FLUTTER_PLUGIN_EXPORT const char *get_name_for_entity(void *const assetManager, const EntityId entityId);
 	FLUTTER_PLUGIN_EXPORT const EntityId find_child_entity_by_name(void *const assetManager, const EntityId parent, const char* name);
+	FLUTTER_PLUGIN_EXPORT void set_recording(void *const viewer, bool recording);
 	FLUTTER_PLUGIN_EXPORT void ios_dummy();
 	FLUTTER_PLUGIN_EXPORT void flutter_filament_free(void *ptr);
 #ifdef __cplusplus

ios/src/FilamentViewer.cpp

@@ -55,6 +55,9 @@
 #include <utils/NameComponentManager.h>
 
 #include <imageio/ImageDecoder.h>
+#include <imageio/ImageEncoder.h>
+#include <image/ColorTransform.h>
+
 
 #include "math.h"
 
@@ -70,6 +73,9 @@
 #include <ktxreader/Ktx2Reader.h>
 
 #include <iostream>
+#include <streambuf>
+#include <sstream>
+#include <istream>
 #include <fstream>
 
 #include <mutex>
@@ -80,6 +86,7 @@
 #include "StreamBufferAdapter.hpp"
 #include "material/image.h"
 #include "TimeIt.hpp"
+#include "ThreadPool.hpp"
 
 using namespace filament;
 using namespace filament::math;
@@ -116,6 +123,8 @@ namespace polyvox
       : _resourceLoaderWrapper(resourceLoaderWrapper)
   {
 
+    _tp = new flutter_filament::ThreadPool(1);
+
     ASSERT_POSTCONDITION(_resourceLoaderWrapper != nullptr, "Resource loader must be non-null");
 
 #if TARGET_OS_IPHONE
@@ -1001,36 +1010,76 @@ namespace polyvox
       cam.lookAt(eye, target, upward);
     }
 
-    // // TODO - this was an experiment but probably useful to keep for debugging
-    // // if pixelBuffer is provided, we will copy the framebuffer into the pixelBuffer.
-    // if (pixelBuffer)
-    // {
-    //   auto pbd = Texture::PixelBufferDescriptor(
-    //       pixelBuffer, size_t(1024 * 768 * 4),
-    //       Texture::Format::RGBA,
-    //       Texture::Type::BYTE, nullptr, callback, data);
-
-    //   _renderer->beginFrame(_swapChain, 0);
-    //   _renderer->render(_view);
-    //   _renderer->readPixels(0, 0, 1024, 768, std::move(pbd));
-    //   _renderer->endFrame();
-    // }
-    // else
-    // {
     // Render the scene, unless the renderer wants to skip the frame.
     if (_renderer->beginFrame(_swapChain, frameTimeInNanos))
     {
       _renderer->render(_view);
+
+      if(_recording) {
+        Viewport const &vp = _view->getViewport();
+        size_t pixelBufferSize = vp.width * vp.height * 4;
+        auto* pixelBuffer = new uint8_t[pixelBufferSize];
+        auto callback = [](void *buf, size_t size, void *data) {
+          ((FilamentViewer*)data)->savePng(buf, size);
+        };
+
+        auto pbd = Texture::PixelBufferDescriptor(
+            pixelBuffer, pixelBufferSize,
+            Texture::Format::RGBA,
+            Texture::Type::UBYTE, nullptr, callback, this);
+    
+        _renderer->readPixels(_rt, 0, 0, vp.width, vp.height, std::move(pbd));
+      }
+      
       _renderer->endFrame();
     }
     else
     {
       _skippedFrames++;
-
-      // std::cout << "Skipped" << std::endl;
-      // skipped frame
     }
-    // }
+  }
+
+  void FilamentViewer::savePng(void* buf, size_t size) {
+
+     std::packaged_task<void()> lambda([=]() mutable {
+      
+      Viewport const &vp = _view->getViewport();
+
+      int digits = 6;
+      std::ostringstream stringStream;
+      stringStream << "/tmp/output_";
+      stringStream << std::setfill('0') << std::setw(digits);
+      stringStream << std::to_string(_frameNumber);
+      stringStream << ".png";
+
+      std::string filename = stringStream.str();
+
+      ofstream wf(filename, ios::out | ios::binary);
+      // Log("%d %d %d", vp.width, vp.height, size);
+      LinearImage image(toLinearWithAlpha<uint8_t>(vp.width, vp.height, vp.width * 4,
+                              static_cast<uint8_t*>(buf)));
+
+      auto result = image::ImageEncoder::encode(
+        wf, image::ImageEncoder::Format::PNG, image, std::string(""), std::string("")
+      );
+
+      delete[] static_cast<uint8_t*>(buf);
+      _frameNumber++;
+
+      if(!result) {
+        Log("Failed to encode");
+      }
+      
+      wf.close();
+      if(!wf.good()) {
+        Log("Write failed!");
+      }
+     });
+     _tp->add_task(lambda);
+  }
+
+  void FilamentViewer::setRecording(bool recording) {
+    this->_recording = recording;
   }
 
   void FilamentViewer::updateViewportAndCameraProjection(
@@ -1177,16 +1226,13 @@ namespace polyvox
   void FilamentViewer::_createManipulator()
   {
     Camera &cam = _view->getCamera();
-    auto &tm = _engine->getTransformManager();
-    auto transformInstance = tm.getInstance(cam.getEntity());
-    auto transform = tm.getTransform(transformInstance);
     
     math::double3 home = cam.getPosition();
     math::double3 up = cam.getUpVector();
     auto fv = cam.getForwardVector();
     math::double3 target = home + fv;
     Viewport const &vp = _view->getViewport();
-    Log("Creating manipulator for viewport size %dx%d at home %f %f %f, fv %f %f %f, up %f %f %f target %f %f %f (norm %f) with _zoomSpeed %f", vp.width, vp.height, home[0], home[1], home[2], fv[0], fv[1], fv[2], up[0], up[1], up[2], target[0], target[1], target[2], norm(home), _zoomSpeed);
+    // Log("Creating manipulator for viewport size %dx%d at home %f %f %f, fv %f %f %f, up %f %f %f target %f %f %f (norm %f) with _zoomSpeed %f", vp.width, vp.height, home[0], home[1], home[2], fv[0], fv[1], fv[2], up[0], up[1], up[2], target[0], target[1], target[2], norm(home), _zoomSpeed);
 
     _manipulator = Manipulator<double>::Builder()
                        .viewport(vp.width, vp.height)

ios/src/FlutterFilamentApi.cpp

@@ -483,6 +483,10 @@ extern "C"
         return ((AssetManager *)assetManager)->getNameForEntity(entityId);
     }
 
+    FLUTTER_PLUGIN_EXPORT void set_recording(void *const viewer, bool recording) {
+        ((FilamentViewer*)viewer)->setRecording(recording);
+    }
+
     FLUTTER_PLUGIN_EXPORT void ios_dummy()
     {
         Log("Dummy called");

lib/filament_controller.dart

@@ -492,4 +492,10 @@ abstract class FilamentController {
   ///
   Future<FilamentEntity> getChildEntity(
       FilamentEntity parent, String childName);
+
+  ///
+  /// If [recording] is set to true, each frame the framebuffer/texture will be written to /tmp/output_*.png.
+  /// This will impact performance; handle with care.
+  ///
+  Future setRecording(bool recording);
 }

lib/filament_controller_ffi.dart

@@ -1207,4 +1207,9 @@ class FilamentControllerFFI extends FilamentController {
       calloc.free(childNamePtr);
     }
   }
+
+  @override
+  Future setRecording(bool recording) async {
+    set_recording(_viewer!, recording);
+  }
 }

lib/generated_bindings.dart

@@ -746,6 +746,13 @@ external int find_child_entity_by_name(
   ffi.Pointer<ffi.Char> name,
 );
 
+@ffi.Native<ffi.Void Function(ffi.Pointer<ffi.Void>, ffi.Bool)>(
+    symbol: 'set_recording', assetId: 'flutter_filament_plugin')
+external void set_recording(
+  ffi.Pointer<ffi.Void> viewer,
+  bool recording,
+);
+
 @ffi.Native<ffi.Void Function()>(
     symbol: 'ios_dummy', assetId: 'flutter_filament_plugin')
 external void ios_dummy();