Skip to content

Commit

Permalink
rotate ID3D11Texture2D (rustdesk#9772)
Browse files Browse the repository at this point in the history 
* Rotate ID3D11Texture2D after duplication with d3d11 video processor.
* If display is not rotated, nothing will be created; If the rotation
  fails, it will use the old fallback logic

TODO:
* If changing from Landscape to Landscape(flipped) during capture, the resolution is
  not changed, video service fallback to gdi directly.

Signed-off-by: 21pages <[email protected]>
  • Loading branch information
@21pages
21pages authored Nov 8, 2024
1 parent 7978e03 commit 740c535
Show file tree
Hide file tree
Showing 3 changed files with 236 additions and 15 deletions.
242 changes: 228 additions & 14 deletions libs/scrap/src/dxgi/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ use winapi::{
shared::{
dxgi::*,
dxgi1_2::*,
dxgiformat::DXGI_FORMAT_B8G8R8A8_UNORM,
dxgitype::*,
minwindef::{DWORD, FALSE, TRUE, UINT},
ntdef::LONG,
windef::HMONITOR,
windef::{HMONITOR, RECT},
winerror::*,
// dxgiformat::{DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_420_OPAQUE},
},
Expand Down Expand Up @@ -57,6 +58,7 @@ pub struct Capturer {
saved_raw_data: Vec<u8>, // for faster compare and copy
output_texture: bool,
adapter_desc1: DXGI_ADAPTER_DESC1,
rotate: Rotate,
}

impl Capturer {
Expand Down Expand Up @@ -151,6 +153,7 @@ impl Capturer {
(*duplication).GetDesc(&mut desc);
}
}
let rotate = Self::create_rotations(device.0, context.0, &display);

Ok(Capturer {
device,
Expand All @@ -168,9 +171,143 @@ impl Capturer {
saved_raw_data: Vec::new(),
output_texture: false,
adapter_desc1,
rotate,
})
}

fn create_rotations(
device: *mut ID3D11Device,
context: *mut ID3D11DeviceContext,
display: &Display,
) -> Rotate {
let mut video_context: *mut ID3D11VideoContext = ptr::null_mut();
let mut video_device: *mut ID3D11VideoDevice = ptr::null_mut();
let mut video_processor_enum: *mut ID3D11VideoProcessorEnumerator = ptr::null_mut();
let mut video_processor: *mut ID3D11VideoProcessor = ptr::null_mut();
let processor_rotation = match display.rotation() {
DXGI_MODE_ROTATION_ROTATE90 => Some(D3D11_VIDEO_PROCESSOR_ROTATION_90),
DXGI_MODE_ROTATION_ROTATE180 => Some(D3D11_VIDEO_PROCESSOR_ROTATION_180),
DXGI_MODE_ROTATION_ROTATE270 => Some(D3D11_VIDEO_PROCESSOR_ROTATION_270),
_ => None,
};
if let Some(processor_rotation) = processor_rotation {
println!("create rotations");
if !device.is_null() && !context.is_null() {
unsafe {
(*context).QueryInterface(
&IID_ID3D11VideoContext,
&mut video_context as *mut *mut _ as *mut *mut _,
);
if !video_context.is_null() {
(*device).QueryInterface(
&IID_ID3D11VideoDevice,
&mut video_device as *mut *mut _ as *mut *mut _,
);
if !video_device.is_null() {
let (input_width, input_height) = match display.rotation() {
DXGI_MODE_ROTATION_ROTATE90 | DXGI_MODE_ROTATION_ROTATE270 => {
(display.height(), display.width())
}
_ => (display.width(), display.height()),
};
let (output_width, output_height) = (display.width(), display.height());
let content_desc = D3D11_VIDEO_PROCESSOR_CONTENT_DESC {
InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
InputFrameRate: DXGI_RATIONAL {
Numerator: 30,
Denominator: 1,
},
InputWidth: input_width as _,
InputHeight: input_height as _,
OutputFrameRate: DXGI_RATIONAL {
Numerator: 30,
Denominator: 1,
},
OutputWidth: output_width as _,
OutputHeight: output_height as _,
Usage: D3D11_VIDEO_USAGE_PLAYBACK_NORMAL,
};
(*video_device).CreateVideoProcessorEnumerator(
&content_desc,
&mut video_processor_enum,
);
if !video_processor_enum.is_null() {
let mut caps: D3D11_VIDEO_PROCESSOR_CAPS = mem::zeroed();
if S_OK == (*video_processor_enum).GetVideoProcessorCaps(&mut caps)
{
if caps.FeatureCaps
& D3D11_VIDEO_PROCESSOR_FEATURE_CAPS_ROTATION
!= 0
{
(*video_device).CreateVideoProcessor(
video_processor_enum,
0,
&mut video_processor,
);
if !video_processor.is_null() {
(*video_context).VideoProcessorSetStreamRotation(
video_processor,
0,
TRUE,
processor_rotation,
);
(*video_context)
.VideoProcessorSetStreamAutoProcessingMode(
video_processor,
0,
FALSE,
);
(*video_context).VideoProcessorSetStreamFrameFormat(
video_processor,
0,
D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
);
(*video_context).VideoProcessorSetStreamSourceRect(
video_processor,
0,
TRUE,
&RECT {
left: 0,
top: 0,
right: input_width as _,
bottom: input_height as _,
},
);
(*video_context).VideoProcessorSetStreamDestRect(
video_processor,
0,
TRUE,
&RECT {
left: 0,
top: 0,
right: output_width as _,
bottom: output_height as _,
},
);
}
}
}
}
}
}
}
}
}

let video_context = ComPtr(video_context);
let video_device = ComPtr(video_device);
let video_processor_enum = ComPtr(video_processor_enum);
let video_processor = ComPtr(video_processor);
let rotated_texture = ComPtr(ptr::null_mut());
Rotate {
video_context,
video_device,
video_processor_enum,
video_processor,
texture: (rotated_texture, false),
}
}

pub fn is_gdi(&self) -> bool {
self.gdi_capturer.is_some()
}
Expand Down Expand Up @@ -253,17 +390,7 @@ impl Capturer {

pub fn frame<'a>(&'a mut self, timeout: UINT) -> io::Result<Frame<'a>> {
if self.output_texture {
let rotation = match self.display.rotation() {
DXGI_MODE_ROTATION_IDENTITY | DXGI_MODE_ROTATION_UNSPECIFIED => 0,
DXGI_MODE_ROTATION_ROTATE90 => 90,
DXGI_MODE_ROTATION_ROTATE180 => 180,
DXGI_MODE_ROTATION_ROTATE270 => 270,
_ => {
// Unsupported rotation, try anyway
0
}
};
Ok(Frame::Texture((self.get_texture(timeout)?, rotation)))
Ok(Frame::Texture(self.get_texture(timeout)?))
} else {
let width = self.width;
let height = self.height;
Expand Down Expand Up @@ -338,7 +465,7 @@ impl Capturer {
}
}

fn get_texture(&mut self, timeout: UINT) -> io::Result<*mut c_void> {
fn get_texture(&mut self, timeout: UINT) -> io::Result<(*mut c_void, usize)> {
unsafe {
if self.duplication.0.is_null() {
return Err(std::io::ErrorKind::AddrNotAvailable.into());
Expand All @@ -362,7 +489,86 @@ impl Capturer {
);
let texture = ComPtr(texture);
self.texture = texture;
Ok(self.texture.0 as *mut c_void)

let mut final_texture = self.texture.0 as *mut c_void;
let mut rotation = match self.display.rotation() {
DXGI_MODE_ROTATION_ROTATE90 => 90,
DXGI_MODE_ROTATION_ROTATE180 => 180,
DXGI_MODE_ROTATION_ROTATE270 => 270,
_ => 0,
};
if rotation != 0
&& !self.texture.is_null()
&& !self.rotate.video_context.is_null()
&& !self.rotate.video_device.is_null()
&& !self.rotate.video_processor_enum.is_null()
&& !self.rotate.video_processor.is_null()
{
let mut desc: D3D11_TEXTURE2D_DESC = mem::zeroed();
(*self.texture.0).GetDesc(&mut desc);
if rotation == 90 || rotation == 270 {
let tmp = desc.Width;
desc.Width = desc.Height;
desc.Height = tmp;
}
if !self.rotate.texture.1 {
self.rotate.texture.1 = true;
let mut rotated_texture: *mut ID3D11Texture2D = ptr::null_mut();
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
(*self.device.0).CreateTexture2D(&desc, ptr::null(), &mut rotated_texture);
self.rotate.texture.0 = ComPtr(rotated_texture);
}
if !self.rotate.texture.0.is_null()
&& desc.Width == self.width as u32
&& desc.Height == self.height as u32
{
let input_view_desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC {
FourCC: 0,
ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D,
Texture2D: D3D11_TEX2D_VPIV {
ArraySlice: 0,
MipSlice: 0,
},
};
let mut input_view = ptr::null_mut();
(*self.rotate.video_device.0).CreateVideoProcessorInputView(
self.texture.0 as *mut _,
self.rotate.video_processor_enum.0 as *mut _,
&input_view_desc,
&mut input_view,
);
if !input_view.is_null() {
let input_view = ComPtr(input_view);
let mut output_view_desc: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC =
mem::zeroed();
output_view_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
output_view_desc.u.Texture2D_mut().MipSlice = 0;
let mut output_view = ptr::null_mut();
(*self.rotate.video_device.0).CreateVideoProcessorOutputView(
self.rotate.texture.0 .0 as *mut _,
self.rotate.video_processor_enum.0 as *mut _,
&output_view_desc,
&mut output_view,
);
if !output_view.is_null() {
let output_view = ComPtr(output_view);
let mut stream_data: D3D11_VIDEO_PROCESSOR_STREAM = mem::zeroed();
stream_data.Enable = TRUE;
stream_data.pInputSurface = input_view.0;
(*self.rotate.video_context.0).VideoProcessorBlt(
self.rotate.video_processor.0,
output_view.0,
0,
1,
&stream_data,
);
final_texture = self.rotate.texture.0 .0 as *mut c_void;
rotation = 0;
}
}
}
}
Ok((final_texture, rotation))
}
}

Expand Down Expand Up @@ -666,3 +872,11 @@ fn wrap_hresult(x: HRESULT) -> io::Result<()> {
})
.into())
}

struct Rotate {
video_context: ComPtr<ID3D11VideoContext>,
video_device: ComPtr<ID3D11VideoDevice>,
video_processor_enum: ComPtr<ID3D11VideoProcessorEnumerator>,
video_processor: ComPtr<ID3D11VideoProcessor>,
texture: (ComPtr<ID3D11Texture2D>, bool),
}
8 changes: 7 additions & 1 deletion src/server/connection.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3838,6 +3838,12 @@ mod raii {
let mut lock = SESSIONS.lock().unwrap();
let contains = lock.contains_key(&key);
if contains {
// No two remote connections with the same session key, just for ensure.
let is_remote = AUTHED_CONNS
.lock()
.unwrap()
.iter()
.any(|c| c.0 == conn_id && c.1 == AuthConnType::Remote);
// If there are 2 connections with the same peer_id and session_id, a remote connection and a file transfer or port forward connection,
// If any of the connections is closed allowing retry, this will not be called;
// If the file transfer/port forward connection is closed with no retry, the session should be kept for remote control menu action;
Expand All @@ -3847,7 +3853,7 @@ mod raii {
.unwrap()
.iter()
.any(|c| c.0 != conn_id && c.2 == key && c.1 == AuthConnType::Remote);
if !another_remote {
if is_remote || !another_remote {
lock.remove(&key);
log::info!("remove session");
} else {
Expand Down
1 change: 1 addition & 0 deletions src/server/video_service.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -971,6 +971,7 @@ fn handle_one_frame(
}
match e.to_string().as_str() {
scrap::codec::ENCODE_NEED_SWITCH => {
encoder.disable();
log::error!("switch due to encoder need switch");
bail!("SWITCH");
}
Expand Down

0 comments on commit 740c535

Please sign in to comment.