fix(server): use desktop size for RFX channel size (#756)

Author: elmarco

crates/ironrdp-server/src/encoder/mod.rs

@@ -47,7 +47,7 @@ impl UpdateEncoder {
             BitmapUpdater::Bitmap(BitmapHandler::new())
         } else if remotefx.is_some() {
             let (algo, id) = remotefx.unwrap();
-            BitmapUpdater::RemoteFx(RemoteFxHandler::new(algo, id))
+            BitmapUpdater::RemoteFx(RemoteFxHandler::new(algo, id, desktop_size))
         } else {
             BitmapUpdater::None(NoneHandler)
         };
@@ -68,6 +68,7 @@ impl UpdateEncoder {
 
     pub(crate) fn set_desktop_size(&mut self, size: DesktopSize) {
         self.desktop_size = size;
+        self.bitmap_updater.set_desktop_size(size);
     }
 
     fn rgba_pointer(ptr: RGBAPointer) -> Result<UpdateFragmenter> {
@@ -181,6 +182,12 @@ impl BitmapUpdater {
             Self::RemoteFx(up) => up.handle(bitmap),
         }
     }
+
+    fn set_desktop_size(&mut self, size: DesktopSize) {
+        if let Self::RemoteFx(up) = self {
+            up.set_desktop_size(size)
+        }
+    }
 }
 
 trait BitmapUpdateHandler {
@@ -246,28 +253,36 @@ impl BitmapUpdateHandler for BitmapHandler {
 struct RemoteFxHandler {
     remotefx: RfxEncoder,
     codec_id: u8,
+    desktop_size: Option<DesktopSize>,
 }
 
 impl RemoteFxHandler {
-    fn new(algo: EntropyBits, codec_id: u8) -> Self {
+    fn new(algo: EntropyBits, codec_id: u8, desktop_size: DesktopSize) -> Self {
         Self {
             remotefx: RfxEncoder::new(algo),
+            desktop_size: Some(desktop_size),
             codec_id,
         }
     }
+
+    fn set_desktop_size(&mut self, size: DesktopSize) {
+        self.desktop_size = Some(size);
+    }
 }
 
 impl BitmapUpdateHandler for RemoteFxHandler {
     fn handle(&mut self, bitmap: &BitmapUpdate) -> Result<UpdateFragmenter> {
         let mut buffer = vec![0; bitmap.data.len()];
         let len = loop {
-            match self.remotefx.encode(bitmap, buffer.as_mut_slice()) {
+            match self
+                .remotefx
+                .encode(bitmap, buffer.as_mut_slice(), self.desktop_size.take())
+            {
                 Err(e) => match e.kind() {
                     ironrdp_core::EncodeErrorKind::NotEnoughBytes { .. } => {
                         buffer.resize(buffer.len() * 2, 0);
                         debug!("encoder buffer resized to: {}", buffer.len() * 2);
                     }
-
                     _ => Err(e).context("RemoteFX encode error")?,
                 },
                 Ok(len) => break len,

crates/ironrdp-server/src/encoder/rfx.rs

@@ -1,3 +1,4 @@
+use ironrdp_acceptor::DesktopSize;
 use ironrdp_core::{cast_length, other_err, Encode, EncodeResult};
 use ironrdp_graphics::color_conversion::to_64x64_ycbcr_tile;
 use ironrdp_graphics::rfx_encode_component;
@@ -25,29 +26,34 @@ impl RfxEncoder {
         Self { entropy_algorithm }
     }
 
-    pub(crate) fn encode(&mut self, bitmap: &BitmapUpdate, output: &mut [u8]) -> EncodeResult<usize> {
+    pub(crate) fn encode(
+        &mut self,
+        bitmap: &BitmapUpdate,
+        output: &mut [u8],
+        desktop_size: Option<DesktopSize>,
+    ) -> EncodeResult<usize> {
         let mut cursor = WriteCursor::new(output);
-
-        let width = bitmap.width.get();
-        let height = bitmap.height.get();
         let entropy_algorithm = self.entropy_algorithm;
 
         // header messages
-        // FIXME: skip if unnecessary?
-        Block::Sync(SyncPdu).encode(&mut cursor)?;
-        let context = rfx::ContextPdu {
-            flags: OperatingMode::IMAGE_MODE,
-            entropy_algorithm,
-        };
-        Block::CodecChannel(CodecChannel::Context(context)).encode(&mut cursor)?;
-
-        let channels = ChannelsPdu(vec![RfxChannel {
-            width: cast_length!("width", width)?,
-            height: cast_length!("height", height)?,
-        }]);
-        Block::Channels(channels).encode(&mut cursor)?;
-
-        Block::CodecVersions(CodecVersionsPdu).encode(&mut cursor)?;
+        if let Some(desktop_size) = desktop_size {
+            let width = desktop_size.width;
+            let height = desktop_size.height;
+            Block::Sync(SyncPdu).encode(&mut cursor)?;
+            let context = rfx::ContextPdu {
+                flags: OperatingMode::IMAGE_MODE,
+                entropy_algorithm,
+            };
+            Block::CodecChannel(CodecChannel::Context(context)).encode(&mut cursor)?;
+
+            let channels = ChannelsPdu(vec![RfxChannel {
+                width: cast_length!("width", width)?,
+                height: cast_length!("height", height)?,
+            }]);
+            Block::Channels(channels).encode(&mut cursor)?;
+
+            Block::CodecVersions(CodecVersionsPdu).encode(&mut cursor)?;
+        }
 
         // data messages
         let frame_begin = FrameBeginPdu {