1
1
use core:: fmt;
2
2
use core:: num:: NonZeroU16 ;
3
+ use core:: time:: Duration ;
4
+ use std:: collections:: HashMap ;
5
+ use std:: time:: Instant ;
3
6
4
7
use anyhow:: { Context , Result } ;
5
8
use ironrdp_acceptor:: DesktopSize ;
@@ -22,6 +25,8 @@ pub(crate) mod rfx;
22
25
23
26
pub ( crate ) use fast_path:: * ;
24
27
28
+ const VIDEO_HINT_FPS : usize = 5 ;
29
+
25
30
#[ derive( Debug , Copy , Clone , PartialEq , Eq ) ]
26
31
#[ repr( u8 ) ]
27
32
enum CodecId {
@@ -57,35 +62,42 @@ pub(crate) struct UpdateEncoder {
57
62
desktop_size : DesktopSize ,
58
63
framebuffer : Option < Framebuffer > ,
59
64
bitmap_updater : BitmapUpdater ,
65
+ video_updater : Option < BitmapUpdater > ,
66
+ region_update_times : HashMap < Rect , Vec < Instant > > ,
60
67
}
61
68
62
69
impl fmt:: Debug for UpdateEncoder {
63
70
fn fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
64
71
f. debug_struct ( "UpdateEncoder" )
65
72
. field ( "bitmap_update" , & self . bitmap_updater )
73
+ . field ( "video_updater" , & self . video_updater )
66
74
. finish ( )
67
75
}
68
76
}
69
77
70
78
impl UpdateEncoder {
71
79
#[ cfg_attr( feature = "__bench" , visibility:: make( pub ) ) ]
72
80
pub ( crate ) fn new ( desktop_size : DesktopSize , surface_flags : CmdFlags , codecs : UpdateEncoderCodecs ) -> Self {
73
- let bitmap_updater = if surface_flags. contains ( CmdFlags :: SET_SURFACE_BITS ) {
81
+ let ( bitmap_updater, video_updater ) = if surface_flags. contains ( CmdFlags :: SET_SURFACE_BITS ) {
74
82
let mut bitmap = BitmapUpdater :: None ( NoneHandler ) ;
83
+ let mut video = None ;
75
84
76
85
if let Some ( ( algo, id) ) = codecs. remotefx {
77
86
bitmap = BitmapUpdater :: RemoteFx ( RemoteFxHandler :: new ( algo, id, desktop_size) ) ;
87
+ video = Some ( bitmap. clone ( ) ) ;
78
88
}
79
89
80
- bitmap
90
+ ( bitmap, video )
81
91
} else {
82
- BitmapUpdater :: Bitmap ( BitmapHandler :: new ( ) )
92
+ ( BitmapUpdater :: Bitmap ( BitmapHandler :: new ( ) ) , None )
83
93
} ;
84
94
85
95
Self {
86
96
desktop_size,
87
97
framebuffer : None ,
88
98
bitmap_updater,
99
+ video_updater,
100
+ region_update_times : HashMap :: new ( ) ,
89
101
}
90
102
}
91
103
@@ -157,6 +169,35 @@ impl UpdateEncoder {
157
169
Ok ( UpdateFragmenter :: new ( UpdateCode :: PositionPointer , encode_vec ( & pos) ?) )
158
170
}
159
171
172
+ // This is a very naive heuristic for detecting video regions
173
+ // based on the number of updates in the last second.
174
+ // Feel free to improve it! :)
175
+ fn diff_hints ( & mut self , now : Instant , off_x : usize , off_y : usize , regions : Vec < Rect > ) -> Vec < HintRect > {
176
+ // keep the updates from the last second
177
+ for ( _region, ts) in self . region_update_times . iter_mut ( ) {
178
+ ts. retain ( |ts| now - * ts < Duration :: from_millis ( 1000 ) ) ;
179
+ }
180
+ self . region_update_times . retain ( |_, times| !times. is_empty ( ) ) ;
181
+
182
+ let mut diffs = Vec :: new ( ) ;
183
+ for rect in regions {
184
+ let rect_root = rect. clone ( ) . add_xy ( off_x, off_y) ;
185
+ let entry = self . region_update_times . entry ( rect_root) . or_default ( ) ;
186
+ entry. push ( now) ;
187
+
188
+ let hint = if entry. len ( ) >= VIDEO_HINT_FPS {
189
+ HintType :: Video
190
+ } else {
191
+ HintType :: Image
192
+ } ;
193
+
194
+ let diff = HintRect :: new ( rect, hint) ;
195
+ diffs. push ( diff) ;
196
+ }
197
+
198
+ diffs
199
+ }
200
+
160
201
fn bitmap_diffs ( & mut self , bitmap : & BitmapUpdate ) -> Vec < Rect > {
161
202
const USE_DIFFS : bool = true ;
162
203
@@ -205,21 +246,46 @@ impl UpdateEncoder {
205
246
}
206
247
}
207
248
208
- async fn bitmap ( & mut self , bitmap : BitmapUpdate ) -> Result < UpdateFragmenter > {
249
+ async fn bitmap ( & mut self , bitmap : BitmapUpdate , hint : HintType ) -> Result < UpdateFragmenter > {
250
+ let updater = match hint {
251
+ HintType :: Image => & self . bitmap_updater ,
252
+ HintType :: Video => {
253
+ trace ! ( ?bitmap, "Encoding with video hint" ) ;
254
+ self . video_updater . as_ref ( ) . unwrap_or ( & self . bitmap_updater )
255
+ }
256
+ } ;
209
257
// Clone to satisfy spawn_blocking 'static requirement
210
258
// this should be cheap, even if using bitmap, since vec![] will be empty
211
- let mut updater = self . bitmap_updater . clone ( ) ;
259
+ let mut updater = updater . clone ( ) ;
212
260
tokio:: task:: spawn_blocking ( move || time_warn ! ( "Encoding bitmap" , 10 , updater. handle( & bitmap) ) )
213
261
. await
214
262
. unwrap ( )
215
263
}
216
264
}
217
265
266
+ #[ derive( Copy , Clone , Debug ) ]
267
+ enum HintType {
268
+ Image ,
269
+ Video ,
270
+ }
271
+
272
+ #[ derive( Debug ) ]
273
+ struct HintRect {
274
+ rect : Rect ,
275
+ hint : HintType ,
276
+ }
277
+
278
+ impl HintRect {
279
+ fn new ( rect : Rect , hint : HintType ) -> Self {
280
+ Self { rect, hint }
281
+ }
282
+ }
283
+
218
284
#[ derive( Debug , Default ) ]
219
285
enum State {
220
286
Start ( DisplayUpdate ) ,
221
287
BitmapDiffs {
222
- diffs : Vec < Rect > ,
288
+ diffs : Vec < HintRect > ,
223
289
bitmap : BitmapUpdate ,
224
290
pos : usize ,
225
291
} ,
@@ -244,6 +310,8 @@ impl EncoderIter<'_> {
244
310
State :: Start ( update) => match update {
245
311
DisplayUpdate :: Bitmap ( bitmap) => {
246
312
let diffs = encoder. bitmap_diffs ( & bitmap) ;
313
+ let diffs =
314
+ encoder. diff_hints ( Instant :: now ( ) , usize:: from ( bitmap. x ) , usize:: from ( bitmap. y ) , diffs) ;
247
315
self . state = State :: BitmapDiffs { diffs, bitmap, pos : 0 } ;
248
316
continue ;
249
317
}
@@ -255,12 +323,14 @@ impl EncoderIter<'_> {
255
323
DisplayUpdate :: Resize ( _) => return None ,
256
324
} ,
257
325
State :: BitmapDiffs { diffs, bitmap, pos } => {
258
- let Some ( rect) = diffs. get ( pos) else {
326
+ let Some ( diff) = diffs. get ( pos) else {
327
+ let diffs = diffs. into_iter ( ) . map ( |diff| diff. rect ) . collect :: < Vec < _ > > ( ) ;
259
328
encoder. bitmap_update_framebuffer ( bitmap, & diffs) ;
260
329
self . state = State :: Ended ;
261
330
return None ;
262
331
} ;
263
- let Rect { x, y, width, height } = * rect;
332
+
333
+ let Rect { x, y, width, height } = diff. rect ;
264
334
let Some ( sub) = bitmap. sub (
265
335
u16:: try_from ( x) . unwrap ( ) ,
266
336
u16:: try_from ( y) . unwrap ( ) ,
@@ -270,12 +340,13 @@ impl EncoderIter<'_> {
270
340
warn ! ( "Failed to extract bitmap subregion" ) ;
271
341
return None ;
272
342
} ;
343
+ let hint = diff. hint ;
273
344
self . state = State :: BitmapDiffs {
274
345
diffs,
275
346
bitmap,
276
347
pos : pos + 1 ,
277
348
} ;
278
- encoder. bitmap ( sub) . await
349
+ encoder. bitmap ( sub, hint ) . await
279
350
}
280
351
State :: Ended => return None ,
281
352
} ;
0 commit comments