Skip to content

Commit 68d6698

Browse files
committed
feat(server): add simple heuristic to detect video regions
If a region is updated more than 5x/seconds, use a video updater for encoding. For now, this is RemoteFx, but it will allow future tweaking. Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
1 parent 831f3ce commit 68d6698

File tree

1 file changed

+80
-9
lines changed
  • crates/ironrdp-server/src/encoder

1 file changed

+80
-9
lines changed

crates/ironrdp-server/src/encoder/mod.rs

Lines changed: 80 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
use core::fmt;
22
use core::num::NonZeroU16;
3+
use core::time::Duration;
4+
use std::collections::HashMap;
5+
use std::time::Instant;
36

47
use anyhow::{Context, Result};
58
use ironrdp_acceptor::DesktopSize;
@@ -22,6 +25,8 @@ pub(crate) mod rfx;
2225

2326
pub(crate) use fast_path::*;
2427

28+
const VIDEO_HINT_FPS: usize = 5;
29+
2530
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
2631
#[repr(u8)]
2732
enum CodecId {
@@ -57,35 +62,42 @@ pub(crate) struct UpdateEncoder {
5762
desktop_size: DesktopSize,
5863
framebuffer: Option<Framebuffer>,
5964
bitmap_updater: BitmapUpdater,
65+
video_updater: Option<BitmapUpdater>,
66+
region_update_times: HashMap<Rect, Vec<Instant>>,
6067
}
6168

6269
impl fmt::Debug for UpdateEncoder {
6370
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6471
f.debug_struct("UpdateEncoder")
6572
.field("bitmap_update", &self.bitmap_updater)
73+
.field("video_updater", &self.video_updater)
6674
.finish()
6775
}
6876
}
6977

7078
impl UpdateEncoder {
7179
#[cfg_attr(feature = "__bench", visibility::make(pub))]
7280
pub(crate) fn new(desktop_size: DesktopSize, surface_flags: CmdFlags, codecs: UpdateEncoderCodecs) -> Self {
73-
let bitmap_updater = if surface_flags.contains(CmdFlags::SET_SURFACE_BITS) {
81+
let (bitmap_updater, video_updater) = if surface_flags.contains(CmdFlags::SET_SURFACE_BITS) {
7482
let mut bitmap = BitmapUpdater::None(NoneHandler);
83+
let mut video = None;
7584

7685
if let Some((algo, id)) = codecs.remotefx {
7786
bitmap = BitmapUpdater::RemoteFx(RemoteFxHandler::new(algo, id, desktop_size));
87+
video = Some(bitmap.clone());
7888
}
7989

80-
bitmap
90+
(bitmap, video)
8191
} else {
82-
BitmapUpdater::Bitmap(BitmapHandler::new())
92+
(BitmapUpdater::Bitmap(BitmapHandler::new()), None)
8393
};
8494

8595
Self {
8696
desktop_size,
8797
framebuffer: None,
8898
bitmap_updater,
99+
video_updater,
100+
region_update_times: HashMap::new(),
89101
}
90102
}
91103

@@ -157,6 +169,35 @@ impl UpdateEncoder {
157169
Ok(UpdateFragmenter::new(UpdateCode::PositionPointer, encode_vec(&pos)?))
158170
}
159171

172+
// This is a very naive heuristic for detecting video regions
173+
// based on the number of updates in the last second.
174+
// Feel free to improve it! :)
175+
fn diff_hints(&mut self, now: Instant, off_x: usize, off_y: usize, regions: Vec<Rect>) -> Vec<HintRect> {
176+
// keep the updates from the last second
177+
for (_region, ts) in self.region_update_times.iter_mut() {
178+
ts.retain(|ts| now - *ts < Duration::from_millis(1000));
179+
}
180+
self.region_update_times.retain(|_, times| !times.is_empty());
181+
182+
let mut diffs = Vec::new();
183+
for rect in regions {
184+
let rect_root = rect.clone().add_xy(off_x, off_y);
185+
let entry = self.region_update_times.entry(rect_root).or_default();
186+
entry.push(now);
187+
188+
let hint = if entry.len() >= VIDEO_HINT_FPS {
189+
HintType::Video
190+
} else {
191+
HintType::Image
192+
};
193+
194+
let diff = HintRect::new(rect, hint);
195+
diffs.push(diff);
196+
}
197+
198+
diffs
199+
}
200+
160201
fn bitmap_diffs(&mut self, bitmap: &BitmapUpdate) -> Vec<Rect> {
161202
const USE_DIFFS: bool = true;
162203

@@ -205,21 +246,46 @@ impl UpdateEncoder {
205246
}
206247
}
207248

208-
async fn bitmap(&mut self, bitmap: BitmapUpdate) -> Result<UpdateFragmenter> {
249+
async fn bitmap(&mut self, bitmap: BitmapUpdate, hint: HintType) -> Result<UpdateFragmenter> {
250+
let updater = match hint {
251+
HintType::Image => &self.bitmap_updater,
252+
HintType::Video => {
253+
trace!(?bitmap, "Encoding with video hint");
254+
self.video_updater.as_ref().unwrap_or(&self.bitmap_updater)
255+
}
256+
};
209257
// Clone to satisfy spawn_blocking 'static requirement
210258
// this should be cheap, even if using bitmap, since vec![] will be empty
211-
let mut updater = self.bitmap_updater.clone();
259+
let mut updater = updater.clone();
212260
tokio::task::spawn_blocking(move || time_warn!("Encoding bitmap", 10, updater.handle(&bitmap)))
213261
.await
214262
.unwrap()
215263
}
216264
}
217265

266+
#[derive(Copy, Clone, Debug)]
267+
enum HintType {
268+
Image,
269+
Video,
270+
}
271+
272+
#[derive(Debug)]
273+
struct HintRect {
274+
rect: Rect,
275+
hint: HintType,
276+
}
277+
278+
impl HintRect {
279+
fn new(rect: Rect, hint: HintType) -> Self {
280+
Self { rect, hint }
281+
}
282+
}
283+
218284
#[derive(Debug, Default)]
219285
enum State {
220286
Start(DisplayUpdate),
221287
BitmapDiffs {
222-
diffs: Vec<Rect>,
288+
diffs: Vec<HintRect>,
223289
bitmap: BitmapUpdate,
224290
pos: usize,
225291
},
@@ -244,6 +310,8 @@ impl EncoderIter<'_> {
244310
State::Start(update) => match update {
245311
DisplayUpdate::Bitmap(bitmap) => {
246312
let diffs = encoder.bitmap_diffs(&bitmap);
313+
let diffs =
314+
encoder.diff_hints(Instant::now(), usize::from(bitmap.x), usize::from(bitmap.y), diffs);
247315
self.state = State::BitmapDiffs { diffs, bitmap, pos: 0 };
248316
continue;
249317
}
@@ -255,12 +323,14 @@ impl EncoderIter<'_> {
255323
DisplayUpdate::Resize(_) => return None,
256324
},
257325
State::BitmapDiffs { diffs, bitmap, pos } => {
258-
let Some(rect) = diffs.get(pos) else {
326+
let Some(diff) = diffs.get(pos) else {
327+
let diffs = diffs.into_iter().map(|diff| diff.rect).collect::<Vec<_>>();
259328
encoder.bitmap_update_framebuffer(bitmap, &diffs);
260329
self.state = State::Ended;
261330
return None;
262331
};
263-
let Rect { x, y, width, height } = *rect;
332+
333+
let Rect { x, y, width, height } = diff.rect;
264334
let Some(sub) = bitmap.sub(
265335
u16::try_from(x).unwrap(),
266336
u16::try_from(y).unwrap(),
@@ -270,12 +340,13 @@ impl EncoderIter<'_> {
270340
warn!("Failed to extract bitmap subregion");
271341
return None;
272342
};
343+
let hint = diff.hint;
273344
self.state = State::BitmapDiffs {
274345
diffs,
275346
bitmap,
276347
pos: pos + 1,
277348
};
278-
encoder.bitmap(sub).await
349+
encoder.bitmap(sub, hint).await
279350
}
280351
State::Ended => return None,
281352
};

0 commit comments

Comments
 (0)