Display video on Entity

[LudoTexx]

Hello,

New to swift here, any idea for displaying video on 3d model entity.

I am able do to it by converting the piphkView into an image, in update loop and apply this image into a plane but I lost lot of frames and its really heavy in memory and CPU. (An I need to display it in a window too first, for being able to getting the video frame)

(edit : and its not smooth at all too)

Any experienced Swift dev for help, please?

Thanks in advance.

import HaishinKit
import SwiftUI
import RealityKit

struct PiPHKSwiftUiView: UIViewRepresentable {
    var piphkView = PiPHKView(frame: .zero)
    
    
    @Binding var rtmpStream: RTMPStream

    func makeUIView(context: Context) -> PiPHKView {
        piphkView.backgroundColor = .clear
        piphkView.videoGravity = .resizeAspectFill
       
        return piphkView
    }

    func updateUIView(_ uiView: PiPHKView, context: Context) {
        piphkView.attachStream(rtmpStream)
    }
    
    func asImage() -> UIImage {
        
        let renderer = UIGraphicsImageRenderer(size: piphkView.bounds.size)
        return renderer.image { rendererContext in
            piphkView.drawHierarchy(in: piphkView.bounds, afterScreenUpdates: true)
        }
        
    }
}

import SwiftUI
import RealityKit
import UIKit
import SceneKit

class ImmersiveViewModel: ObservableObject {
    @Published var imgEntity = Entity()
    
    func updateImmersivePicture() {
        let newEntity = createImmersivePicture2(from: GlobalVariables.shared.lfView.asImage())
        newEntity.setParent(imgEntity.parent)
        imgEntity.removeFromParent()
        imgEntity = newEntity
    }
    
     func createImmersivePicture2(from image: UIImage) -> Entity {
        let modelEntity = Entity()
        
        guard let cgImage = image.cgImage else {
            fatalError("Failed to convert UIImage to CGImage")
        }
        
        let textureOptions = TextureResource.CreateOptions(semantic: .color)
        
        guard let texture = try? TextureResource.generate(from: cgImage, options: textureOptions) else {
            fatalError("Failed to create TextureResource from CGImage")
        }
        
        var material = UnlitMaterial()
        material.color = .init(texture: .init(texture))
        
        modelEntity.components.set(
            ModelComponent(mesh: .generatePlane(width: 3, height: 2), materials: [material])
        )
        
        return modelEntity
    }
}

struct ImmersiveView: View {
    
    @StateObject private var viewModel = ImmersiveViewModel()
    
    var body: some View {
        
        RealityView{ content, attachments in
            
            
            let anchorEntity = AnchorEntity(.head)
            anchorEntity.anchoring.trackingMode = .continuous
            
            anchorEntity.position = SIMD3<Float>(0, 0, -3)
            anchorEntity.name = "Head Anchor"
            
            viewModel.imgEntity = viewModel.createImmersivePicture2(from: GlobalVariables.shared.lfView.asImage())
            viewModel.imgEntity.setParent(anchorEntity)

            content.add(anchorEntity)
                            
        } update: { content, attachments in
            DispatchQueue.main.async {
                viewModel.updateImmersivePicture()
                //Thread.sleep(forTimeInterval: 0.25)
            }
        }
        attachments: {
            //
        }
        .edgesIgnoringSafeArea(.all)
        
    }
    
    
}

[shogo4405]

Raw video data can be obtained using the following method. I believe it would be better to create a UIImage or similar from this CMSampleBuffer.

class YourStreamOutput: HKStreamOutput {
  func stream(_ stream: some HKStream, didOutput audio: AVAudioBuffer, when: AVAudioTime) {
  }

  func stream(_ stream: some HKStream, didOutput video: CMSampleBuffer) {
    print(video)
  }
}

var output = YouStreamOutput()
rtmpStream.addOutput(output)