maobo/cup_edit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use imported texture on iOS

Browse Source
This commit is contained in:
Nick Fisher
2024-09-30 14:51:11 +08:00
parent c80c163212
commit 8a94b6a334
11 changed files with 233 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
thermion_dart/lib/src/viewer/src/ffi/src/thermion_viewer_ffi.dart
View File

@@ -2041,9 +2041,10 @@ class ThermionViewerFFI extends ThermionViewer {
final swapChain = Viewer_getSwapChainAt(_viewer!, 0);
Viewer_requestFrameRenderThread(_viewer!, swapChain, callback.nativeFunction);
Viewer_requestFrameRenderThread(
_viewer!, swapChain, callback.nativeFunction);
await completer.future;
await completer.future.timeout(Duration(seconds: 1));
}
Future<Camera> createCamera() async {

4
thermion_dart/native/src/FilamentViewer.cpp
View File

@@ -669,9 +669,6 @@ namespace thermion
{
std::lock_guard lock(_renderMutex);
SwapChain *swapChain;
#if TARGET_OS_IPHONE
swapChain = _engine->createSwapChain((void *)window, filament::backend::SWAP_CHAIN_CONFIG_TRANSPARENT | filament::backend::SWAP_CHAIN_CONFIG_APPLE_CVPIXELBUFFER);
#else
if (window)
{
swapChain = _engine->createSwapChain((void *)window, filament::backend::SWAP_CHAIN_CONFIG_TRANSPARENT | filament::backend::SWAP_CHAIN_CONFIG_READABLE);
@@ -682,7 +679,6 @@ namespace thermion
Log("Created headless swapchain %dx%d.", width, height);
swapChain = _engine->createSwapChain(width, height, filament::backend::SWAP_CHAIN_CONFIG_TRANSPARENT | filament::backend::SWAP_CHAIN_CONFIG_READABLE | filament::SwapChain::CONFIG_HAS_STENCIL_BUFFER);
}
#endif
_swapChains.push_back(swapChain);
return swapChain;
}

55
thermion_dart/native/src/ThermionDartRenderThreadApi.cpp
View File

@@ -91,40 +91,45 @@ public:
void iter()
{
std::unique_lock<std::mutex> lock(_mutex);
if (swapChain)
{
doRender(swapChain);
this->_requestFrameRenderCallback();
swapChain = nullptr;
// Calculate and print FPS
auto currentTime = std::chrono::high_resolution_clock::now();
float deltaTime = std::chrono::duration<float, std::chrono::seconds::period>(currentTime - _lastFrameTime).count();
_lastFrameTime = currentTime;
_frameCount++;
_accumulatedTime += deltaTime;
if (_accumulatedTime >= 1.0f) // Update FPS every second
std::unique_lock<std::mutex> lock(_mutex);
if (swapChain)
{
_fps = _frameCount / _accumulatedTime;
std::cout << "FPS: " << _fps << std::endl;
_frameCount = 0;
_accumulatedTime = 0.0f;
doRender(swapChain);
lock.unlock();
this->_requestFrameRenderCallback();
swapChain = nullptr;
// Calculate and print FPS
auto currentTime = std::chrono::high_resolution_clock::now();
float deltaTime = std::chrono::duration<float, std::chrono::seconds::period>(currentTime - _lastFrameTime).count();
_lastFrameTime = currentTime;
_frameCount++;
_accumulatedTime += deltaTime;
if (_accumulatedTime >= 1.0f) // Update FPS every second
{
_fps = _frameCount / _accumulatedTime;
std::cout << "FPS: " << _fps << std::endl;
_frameCount = 0;
_accumulatedTime = 0.0f;
}
}
}
std::unique_lock<std::mutex> taskLock(_taskMutex);
if (!_tasks.empty())
{
auto task = std::move(_tasks.front());
_tasks.pop_front();
lock.unlock();
taskLock.unlock();
task();
lock.lock();
taskLock.lock();
}
_cv.wait_for(lock, std::chrono::microseconds(1000), [this]
{ return !_tasks.empty() || _stop; });
_cv.wait_for(taskLock, std::chrono::microseconds(1000), [this]
{ return !_tasks.empty() || _stop; });
}
@@ -199,14 +204,13 @@ public:
void setFrameIntervalInMilliseconds(float frameIntervalInMilliseconds)
{
std::unique_lock<std::mutex> lock(_mutex);
_frameIntervalInMicroseconds = static_cast<int>(1000.0f * frameIntervalInMilliseconds);
}
template <class Rt>
auto add_task(std::packaged_task<Rt()> &pt) -> std::future<Rt>
{
std::unique_lock<std::mutex> lock(_mutex);
std::unique_lock<std::mutex> lock(_taskMutex);
auto ret = pt.get_future();
_tasks.push_back([pt = std::make_shared<std::packaged_task<Rt()>>(
std::move(pt))]
@@ -223,6 +227,7 @@ private:
bool _stop = false;
int _frameIntervalInMicroseconds = 1000000 / 60;
std::mutex _mutex;
std::mutex _taskMutex;
std::condition_variable _cv;
void (*_renderCallback)(void *const) = nullptr;
void *_renderCallbackOwner = nullptr;

42
thermion_flutter/thermion_flutter/ios/Classes/SwiftThermionFlutterPlugin.swift
View File

@@ -6,7 +6,7 @@ public class SwiftThermionFlutterPlugin: NSObject, FlutterPlugin {
var registrar : FlutterPluginRegistrar
var registry: FlutterTextureRegistry
var texture: ThermionFlutterTexture?
var textures: [Int64: ThermionFlutterTexture] = [:]
var createdAt = Date()
@@ -116,13 +116,12 @@ public class SwiftThermionFlutterPlugin: NSObject, FlutterPlugin {
instance.resources.removeObject(forKey:rbuf.id)
}
var markTextureFrameAvailable : @convention(c) (UnsafeMutableRawPointer?) -> () = { instancePtr in
let instance:SwiftThermionFlutterPlugin = Unmanaged<SwiftThermionFlutterPlugin>.fromOpaque(instancePtr!).takeUnretainedValue()
if(instance.texture != nil) {
instance.registry.textureFrameAvailable(instance.texture!.flutterTextureId)
var markTextureFrameAvailable: @convention(c) (UnsafeMutableRawPointer?) -> () = { instancePtr in
let instance: SwiftThermionFlutterPlugin = Unmanaged<SwiftThermionFlutterPlugin>.fromOpaque(instancePtr!).takeUnretainedValue()
for (_, texture) in instance.textures {
instance.registry.textureFrameAvailable(texture.flutterTextureId)
}
}
public static func register(with registrar: FlutterPluginRegistrar) {
let _messenger = registrar.messenger();
@@ -151,22 +150,35 @@ public class SwiftThermionFlutterPlugin: NSObject, FlutterPlugin {
result(nil)
case "getSharedContext":
result(nil)
case "markTextureFrameAvailable":
let flutterTextureId = call.arguments as! Int64
registry.textureFrameAvailable(flutterTextureId)
result(nil)
case "createTexture":
let args = call.arguments as! [Any]
let width = args[0] as! Int64
let height = args[1] as! Int64
self.texture = ThermionFlutterTexture(width: width, height: height, registry: registry)
let pixelBufferPtr = unsafeBitCast(self.texture!.pixelBuffer, to:UnsafeRawPointer.self)
let pixelBufferAddress = Int(bitPattern:pixelBufferPtr);
let texture = ThermionFlutterTexture(registry: registry, width: width, height: height)
result([self.texture!.flutterTextureId as Any, nil, pixelBufferAddress])
if texture.texture.metalTextureAddress == -1 {
result(nil)
} else {
textures[texture.flutterTextureId] = texture
result([texture.flutterTextureId, texture.texture.metalTextureAddress, nil])
}
case "destroyTexture":
let texture = self.texture
self.texture = nil
texture?.destroy()
let args = call.arguments as! [Any]
let flutterTextureId = args[0] as! Int64
result(true)
if let texture = textures[flutterTextureId] {
registry.unregisterTexture(flutterTextureId)
texture.destroy()
textures.removeValue(forKey: flutterTextureId)
result(true)
} else {
result(false)
}
default:
result(FlutterMethodNotImplemented)
}

40
thermion_flutter/thermion_flutter/ios/Classes/ThermionFlutterTexture.swift
View File

@@ -4,45 +4,31 @@ import Flutter
public class ThermionFlutterTexture : NSObject, FlutterTexture {
public var pixelBuffer: CVPixelBuffer?
var pixelBufferAttrs = [
kCVPixelBufferPixelFormatTypeKey: NSNumber(value: kCVPixelFormatType_32BGRA),
kCVPixelBufferOpenGLCompatibilityKey: kCFBooleanTrue,
kCVPixelBufferOpenGLESCompatibilityKey: kCFBooleanTrue,
kCVPixelBufferIOSurfacePropertiesKey: [:]
] as CFDictionary
var flutterTextureId: Int64 = -1
var registry: FlutterTextureRegistry?
var registry: FlutterTextureRegistry
var texture: ThermionTextureSwift
init(width:Int64, height:Int64, registry:FlutterTextureRegistry) {
init(registry:FlutterTextureRegistry, width:Int64, height:Int64) {
self.registry = registry
self.texture = ThermionTextureSwift(width:width, height: height)
super.init()
if(CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height),
kCVPixelFormatType_32BGRA, pixelBufferAttrs, &pixelBuffer) != kCVReturnSuccess) {
print("Error allocating pixel buffer")
} else {
self.flutterTextureId = registry.register(self)
}
self.flutterTextureId = registry.register(self)
}
public func copyPixelBuffer() -> Unmanaged<CVPixelBuffer>? {
return Unmanaged.passRetained(pixelBuffer!);
if(self.texture.pixelBuffer == nil) {
return nil
}
return Unmanaged.passRetained(self.texture.pixelBuffer!);
}
public func onTextureUnregistered(_ texture:FlutterTexture) {
print("Texture unregistered")
}
public func destroy() {
if(self.flutterTextureId != -1) {
self.registry!.unregisterTexture(self.flutterTextureId)
}
self.pixelBuffer = nil
self.registry.unregisterTexture(self.flutterTextureId)
self.texture.destroyTexture()
}
}

142
thermion_flutter/thermion_flutter/ios/Classes/ThermionTexture.swift Normal file
View File

@@ -0,0 +1,142 @@
import Foundation
import GLKit
@objc public class ThermionTextureSwift : NSObject {
public var pixelBuffer: CVPixelBuffer?
var pixelBufferAttrs = [
kCVPixelBufferPixelFormatTypeKey: NSNumber(value: kCVPixelFormatType_32ABGR ),
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary
] as [CFString : Any] as CFDictionary
@objc public var cvMetalTextureCache:CVMetalTextureCache?
@objc public var metalDevice:MTLDevice?
@objc public var cvMetalTexture:CVMetalTexture?
@objc public var metalTexture:MTLTexture?
@objc public var metalTextureAddress:Int = -1
@objc override public init() {
}
@objc public init(width:Int64, height:Int64) {
if(self.metalDevice == nil) {
self.metalDevice = MTLCreateSystemDefaultDevice()!
}
// create pixel buffer
if(CVPixelBufferCreate(kCFAllocatorDefault, Int(width), Int(height),
kCVPixelFormatType_32BGRA, pixelBufferAttrs, &pixelBuffer) != kCVReturnSuccess) {
print("Error allocating pixel buffer")
metalTextureAddress = -1;
return
}
if self.cvMetalTextureCache == nil {
let cacheCreationResult = CVMetalTextureCacheCreate(
kCFAllocatorDefault,
nil,
self.metalDevice!,
nil,
&self.cvMetalTextureCache)
if(cacheCreationResult != kCVReturnSuccess) {
print("Error creating Metal texture cache")
metalTextureAddress = -1
return
}
}
let cvret = CVMetalTextureCacheCreateTextureFromImage(
kCFAllocatorDefault,
self.cvMetalTextureCache!,
pixelBuffer!, nil,
MTLPixelFormat.bgra8Unorm,
Int(width), Int(height),
0,
&cvMetalTexture)
if(cvret != kCVReturnSuccess) {
print("Error creating texture from image")
metalTextureAddress = -1
return
}
metalTexture = CVMetalTextureGetTexture(cvMetalTexture!)
let metalTexturePtr = Unmanaged.passRetained(metalTexture!).toOpaque()
metalTextureAddress = Int(bitPattern:metalTexturePtr)
}
@objc public func destroyTexture() {
CVMetalTextureCacheFlush(self.cvMetalTextureCache!, 0)
self.metalTexture = nil
self.cvMetalTexture = nil
self.pixelBuffer = nil
self.metalDevice = nil
self.cvMetalTextureCache = nil
}
@objc public func fillColor() {
CVPixelBufferLockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
let bufferWidth = Int(CVPixelBufferGetWidth(pixelBuffer!))
let bufferHeight = Int(CVPixelBufferGetHeight(pixelBuffer!))
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer!)
guard let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer!) else {
return
}
for row in 0..<bufferHeight {
var pixel = baseAddress + row * bytesPerRow
for col in 0..<bufferWidth {
let blue = pixel
blue.storeBytes(of: 255, as: UInt8.self)
let red = pixel + 1
red.storeBytes(of: 0, as: UInt8.self)
let green = pixel + 2
green.storeBytes(of: 0, as: UInt8.self)
let alpha = pixel + 3
alpha.storeBytes(of: 255, as: UInt8.self)
pixel += 4;
}
}
CVPixelBufferUnlockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
}
@objc public func getTextureBytes() -> NSData? {
guard let texture = self.metalTexture else {
print("Metal texture is not available")
return nil
}
let width = texture.width
let height = texture.height
let bytesPerPixel = 4 // RGBA
let bytesPerRow = width * bytesPerPixel
let byteCount = bytesPerRow * height
var bytes = [UInt8](repeating: 0, count: byteCount)
let region = MTLRegionMake2D(0, 0, width, height)
texture.getBytes(&bytes, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0)
// Swizzle bytes from BGRA to RGBA
for i in stride(from: 0, to: byteCount, by: 4) {
let blue = bytes[i]
let green = bytes[i + 1]
let red = bytes[i + 2]
let alpha = bytes[i + 3]
bytes[i] = red
bytes[i + 1] = green
bytes[i + 2] = blue
bytes[i + 3] = alpha
}
// Convert Swift Data to Objective-C NSData
let nsData = Data(bytes: &bytes, count: byteCount) as NSData
return nsData
}
}

1
thermion_flutter/thermion_flutter/lib/src/widgets/src/thermion_widget.dart
View File

@@ -34,6 +34,7 @@ class ThermionWidget extends StatefulWidget {
{Key? key, this.initial, required this.viewer, this.view, this.options})
: super(key: key);
@override
State<ThermionWidget> createState() => _ThermionWidgetState();
}

3
thermion_flutter/thermion_flutter_ffi/lib/thermion_flutter_ffi.dart
View File

@@ -1,4 +1,3 @@
export 'thermion_flutter_android.dart';
export 'thermion_flutter_macos.dart';
export 'thermion_flutter_windows.dart';
export 'thermion_flutter_ios.dart';
export 'thermion_flutter_texture_backed_platform.dart';

131
thermion_flutter/thermion_flutter_ffi/lib/thermion_flutter_ios.dart
View File

@@ -1,131 +0,0 @@
import 'dart:async';
import 'package:flutter/services.dart';
import 'dart:ffi';
import 'package:thermion_dart/thermion_dart.dart';
import 'package:thermion_dart/src/viewer/src/ffi/thermion_viewer_ffi.dart';
import 'package:thermion_flutter_ffi/thermion_flutter_method_channel_interface.dart';
import 'package:thermion_flutter_platform_interface/thermion_flutter_platform_interface.dart';
import 'package:thermion_flutter_platform_interface/thermion_flutter_texture.dart';
import 'package:logging/logging.dart';
///
/// An implementation of [ThermionFlutterPlatform] that uses
/// Flutter platform channels to create a rendering context,
/// resource loaders, and surface/render target(s).
///
class ThermionFlutterIOS
extends ThermionFlutterMethodChannelInterface {
final _channel = const MethodChannel("dev.thermion.flutter/event");
final _logger = Logger("ThermionFlutterFFI");
ThermionViewerFFI? _viewer;
ThermionFlutterIOS._() {}
RenderTarget? _renderTarget;
SwapChain? _swapChain;
static void registerWith() {
ThermionFlutterPlatform.instance = ThermionFlutterIOS._();
}
final _textures = <ThermionFlutterTexture>{};
bool _creatingTexture = false;
bool _destroyingTexture = false;
bool _resizing = false;
///
/// Create a rendering surface.
///
/// This is internal; unless you are [thermion_*] package developer, don't
/// call this yourself.
///
/// The name here is slightly misleading because we only create
/// a texture render target on macOS and iOS; on Android, we render into
/// a native window derived from a Surface, and on Windows we render into
/// a HWND.
///
/// Currently, this only supports a single "texture" (aka rendering surface)
/// at any given time. If a [ThermionWidget] is disposed, it will call
/// [destroyTexture]; if it is resized, it will call [resizeTexture].
///
/// In future, we probably want to be able to create multiple distinct
/// textures/render targets. This would make it possible to have multiple
/// Flutter Texture widgets, each with its own Filament View attached.
/// The current design doesn't accommodate this (for example, it seems we can
/// only create a single native window from a Surface at any one time).
///
Future<ThermionFlutterTexture?> createTexture(int width, int height) async {
throw Exception("TODO");
// note that when [ThermionWidget] is disposed, we don't destroy the
// texture; instead, we keep it around in case a subsequent call requests
// a texture of the same size.
// if (_textures.length > 1) {
// throw Exception("Multiple textures not yet supported");
// } else if (_textures.length == 1) {
// if (_textures.first.height == physicalHeight &&
// _textures.first.width == physicalWidth) {
// return _textures.first;
// } else {
// await _viewer!.setRendering(false);
// await _swapChain?.destroy();
// await destroyTexture(_textures.first);
// _textures.clear();
// }
// }
// _creatingTexture = true;
// var result = await _channel.invokeMethod("createTexture",
// [physicalWidth, physicalHeight, offsetLeft, offsetLeft]);
// if (result == null || (result[0] == -1)) {
// throw Exception("Failed to create texture");
// }
// final flutterTextureId = result[0] as int?;
// final hardwareTextureId = result[1] as int?;
// final surfaceAddress = result[2] as int?;
// _logger.info(
// "Created texture with flutter texture id ${flutterTextureId}, hardwareTextureId $hardwareTextureId and surfaceAddress $surfaceAddress");
// final texture = ThermionFlutterTexture(flutterTextureId, hardwareTextureId,
// physicalWidth, physicalHeight, surfaceAddress);
// await _viewer?.createSwapChain(physicalWidth, physicalHeight,
// surface: texture.surfaceAddress == null
// ? nullptr
// : Pointer<Void>.fromAddress(texture.surfaceAddress!));
// if (texture.hardwareTextureId != null) {
// if (_renderTarget != null) {
// await _renderTarget!.destroy();
// }
// // ignore: unused_local_variable
// _renderTarget = await _viewer?.createRenderTarget(
// physicalWidth, physicalHeight, texture.hardwareTextureId!);
// }
// await _viewer?.updateViewportAndCameraProjection(
// physicalWidth.toDouble(), physicalHeight.toDouble());
// _creatingTexture = false;
// _textures.add(texture);
// return texture;
}
///
/// Called by [ThermionWidget] to resize a texture. Don't call this yourself.
///
@override
Future resizeWindow(
int width,
int height,
int offsetLeft,
int offsetTop,
) async {
throw Exception("Not supported on iOS");
}
}

27
thermion_flutter/thermion_flutter_ffi/lib/thermion_flutter_macos.dart → thermion_flutter/thermion_flutter_ffi/lib/thermion_flutter_texture_backed_platform.dart
View File

@@ -11,18 +11,18 @@ import 'package:logging/logging.dart';
/// Flutter platform channels to create a rendering context,
/// resource loaders, and surface/render target(s).
///
class ThermionFlutterMacOS extends ThermionFlutterMethodChannelInterface {
class ThermionFlutterTextureBackedPlatform extends ThermionFlutterMethodChannelInterface {
final _channel = const MethodChannel("dev.thermion.flutter/event");
final _logger = Logger("ThermionFlutterMacOS");
final _logger = Logger("ThermionFlutterTextureBackedPlatform");
static SwapChain? _swapChain;
ThermionFlutterMacOS._();
ThermionFlutterTextureBackedPlatform._();
static ThermionFlutterMacOS? instance;
static ThermionFlutterTextureBackedPlatform? instance;
static void registerWith() {
instance ??= ThermionFlutterMacOS._();
instance ??= ThermionFlutterTextureBackedPlatform._();
ThermionFlutterPlatform.instance = instance!;
}
@@ -32,15 +32,16 @@ class ThermionFlutterMacOS extends ThermionFlutterMethodChannelInterface {
if (_swapChain != null) {
throw Exception("Only a single swapchain can be created");
}
// this is the headless swap chain
// since we will be using render targets, the actual dimensions don't matter
// this implementation renders directly into a texture/render target
// we still need to create a (headless) swapchain, but the actual dimensions
// don't matter
_swapChain = await viewer.createSwapChain(1, 1);
return viewer;
}
// On desktop platforms, textures are always created
Future<ThermionFlutterTexture?> createTexture(int width, int height) async {
var texture = MacOSMethodChannelFlutterTexture(_channel);
var texture = ThermionFlutterTexture(_channel);
await texture.resize(width, height, 0, 0);
return texture;
}
@@ -65,8 +66,8 @@ class TextureCacheEntry {
}
class MacOSMethodChannelFlutterTexture extends MethodChannelFlutterTexture {
final _logger = Logger("MacOSMethodChannelFlutterTexture");
class ThermionFlutterTexture extends MethodChannelFlutterTexture {
final _logger = Logger("ThermionFlutterTexture");
int flutterId = -1;
int hardwareId = -1;
@@ -75,7 +76,7 @@ class MacOSMethodChannelFlutterTexture extends MethodChannelFlutterTexture {
static final Map<String, List<TextureCacheEntry>> _textureCache = {};
MacOSMethodChannelFlutterTexture(super.channel);
ThermionFlutterTexture(super.channel);
@override
Future<void> resize(
@@ -115,7 +116,7 @@ class MacOSMethodChannelFlutterTexture extends MethodChannelFlutterTexture {
final newEntry = TextureCacheEntry(flutterId, hardwareId, inUse: true);
_textureCache.putIfAbsent(cacheKey, () => []).add(newEntry);
_logger.info(
"Created new MacOS texture: flutter id $flutterId, hardware id $hardwareId");
"Created new texture: flutter id $flutterId, hardware id $hardwareId");
}
// Mark old texture as not in use
@@ -160,7 +161,7 @@ class MacOSMethodChannelFlutterTexture extends MethodChannelFlutterTexture {
}
}
Future<void> _destroyTexture(int flutterId, int hardwareId) async {
Future<void> _destroyTexture(int flutterId, int? hardwareId) async {
try {
await channel.invokeMethod("destroyTexture", [flutterId, hardwareId]);
_logger.info("Destroyed old texture: flutter id $flutterId, hardware id $hardwareId");

4
thermion_flutter/thermion_flutter_ffi/pubspec.yaml
View File

@@ -11,11 +11,11 @@ flutter:
implements: thermion_flutter_platform_interface
platforms:
ios:
dartPluginClass: ThermionFlutterIOS
dartPluginClass: ThermionFlutterTextureBackedPlatform
android:
dartPluginClass: ThermionFlutterAndroid
macos:
dartPluginClass: ThermionFlutterMacOS
dartPluginClass: ThermionFlutterTextureBackedPlatform
windows:
dartPluginClass: ThermionFlutterWindows
dependencies: