x11 + GNOME/Wayland support

Cargo.toml

@@ -24,9 +24,16 @@ rand = "0.8.5"
 reqwest = { version = "0.11", features = ["blocking", "json"] }
 semver = "1.0.7"
 lazy_static = "1.4.0"
+screenshots = { git = "https://github.com/poly000/screenshots-rs", rev = "d96dff76c5f5cbd849d80451f0df8f415f8e5f4b" }
 
 [target.'cfg(windows)'.dependencies]
-winapi = { version = "0.3", features = ["winuser", "winbase", "wingdi", "winnt", "securitybaseapi", "libloaderapi"] }
+winapi = { version = "0.3", features = [
+    "winuser",
+    "wingdi",
+    "winnt",
+    "securitybaseapi",
+    "libloaderapi",
+] }
 
 [build-dependencies]
 winres = "0.1"

README.md

@@ -21,9 +21,16 @@ SVTR原文使用了多个Local/Global Mixing，其中Global Mixing就是Transfor
 *Yas*同样采用PaddleOCR的做法，即MobileNetV3_Small + Global Mixing，相当于将原RNN替换为Transformer。
 
 ## 使用
+### Windows
 - 打开原神，并切换到背包页面，将背包拉到最上面
 - 下载单exe可执行文件，右键管理员运行
 - 扫描过程中，鼠标右键终止
+### Linux
+- 首先请确保自己在x11下或者GNOME/Wayland下（其他wayland de下[会有很坏的性能](https://github.com/poly000/screenshots-rs/blob/d96dff76c5f5cbd849d80451f0df8f415f8e5f4b/src/linux/wayland_screenshot.rs#L109)）
+- 用wine窗口化运行原神（或者全屏+虚拟桌面），打开圣遗物界面，拉到最顶
+- 启动yas
+- Alt+Tab切换到原神窗口，并且在鼠标变为十字后点击一下（还没做窗口聚焦），注意保证原神窗口整体在屏幕内
+- 等待扫描结束。右键中止还没做
 ### 注意
 - 默认4星以下圣遗物不扫描
 - 不是所有窗口比例都支持，推荐16:9的分辨率（如1600x900, 1920x1080, 3840x2160)
@@ -47,8 +54,11 @@ yas --max-row=1
 
 ## 编译
 
+在构建前，请确保安装`Git LFS`，并运行`git lfs pull`。否则[yas在运行时会使用错误的模型](https://github.com/wormtql/yas/pull/102#issuecomment-1375503803)。
+
 ```shell
 # Linux下需要首先安装rustup以及mingw-w64，然后再安装对应的rust target，
+# 构建到Linux需要 `libxdo` 和 `libxcb`
 rustup default stable
 rustup target add x86_64-pc-windows-gnu
 cargo build --release --locked --target=x86_64-pc-windows-gnu

src/capture/mod.rs

@@ -1,192 +1,65 @@
-#[cfg(windows)] extern crate winapi;
-use std::io::Error;
-use std::ffi::{OsStr};
-use std::iter::once;
-use std::os::windows::ffi::OsStrExt;
-use std::ptr::null_mut;
-use std::mem::{size_of, transmute};
+use image::{Rgb, RgbImage};
 
-use winapi::um::winuser::{
-    FindWindowW,
-    GetDC,
-    ReleaseDC,
-    SetThreadDpiAwarenessContext,
-    GetClientRect,
-    SetForegroundWindow
-};
-use winapi::shared::windef::{HWND, HDC, RECT, HBITMAP, DPI_AWARENESS_CONTEXT};
-use winapi::shared::ntdef::NULL;
-use winapi::um::wingdi::{
-    CreateCompatibleDC,
-    DeleteObject,
-    BitBlt,
-    SRCCOPY,
-    CreateCompatibleBitmap,
-    SelectObject,
-    GetObjectW,
-    BITMAP,
-    BITMAPINFOHEADER,
-    BI_RGB,
-    GetDIBits,
-    BITMAPINFO,
-    DIB_RGB_COLORS,
-};
-use winapi::ctypes::{c_void};
-use winapi::um::winbase::{GlobalAlloc, GHND, GlobalLock};
-
-use image::ImageBuffer;
-
-use crate::common::{PixelRect, PixelRectBound};
 use crate::common::color::Color;
-use winapi::shared::windef::DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE;
-use self::winapi::um::wingdi::{GetDeviceCaps, HORZRES};
-use self::winapi::shared::windef::DPI_AWARENESS_CONTEXT_SYSTEM_AWARE;
-
-
-#[cfg(windows)]
-unsafe fn unsafe_capture(rect: &PixelRect) -> Result<Vec<u8>, String> {
-    // SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_SYSTEM_AWARE);
-
-    let dc_window: HDC = GetDC(null_mut());
-
-    let dc_mem: HDC = CreateCompatibleDC(dc_window);
-    if dc_mem.is_null() {
-        return Err(String::from("CreateCompatibleDC Failed"));
-    }
-
-    let hbm: HBITMAP = CreateCompatibleBitmap(dc_window, rect.width, rect.height);
-    if hbm.is_null() {
-        return Err(String::from("CreateCompatibleBitmap failed"));
-    }
-
-    SelectObject(dc_mem, hbm as *mut c_void);
-
-    let result = BitBlt(
-        dc_mem,
-        0,
-        0,
-        rect.width,
-        rect.height,
-        dc_window,
-        rect.left,
-        rect.top,
-        SRCCOPY
-    );
-    if result == 0 {
-        return Err(String::from("BitBlt failed"));
+use crate::common::PixelRect;
+
+/// retures Ok(buf) on success
+/// buf contains pixels in [b:u8, g:u8, r:u8, a:u8] format, as an `[[i32;width];height]`.
+pub fn capture_absolute(
+    PixelRect {
+        left,
+        top,
+        width,
+        height,
+    }: &PixelRect,
+) -> Result<Vec<u8>, String> {
+    let screen = screenshots::Screen::all().ok_or("cannot get DisplayInfo")?[0];
+    let (mut buffer, is_bgra) = screen
+        .capture_area(*left, *top, *width as u32, *height as u32)
+        .ok_or("capture failed")?;
+
+    if !is_bgra {
+        for chunk in buffer.chunks_mut(4) {
+            let temp = chunk[0];
+            chunk[0] = chunk[2];
+            chunk[2] = temp;
+        }
     }
 
-    let mut bitmap: BITMAP = BITMAP {
-        bmBits: 0 as *mut c_void,
-        bmBitsPixel: 0,
-        bmPlanes: 0,
-        bmWidthBytes: 0,
-        bmHeight: 0,
-        bmWidth: 0,
-        bmType: 0,
-    };
-    GetObjectW(
-        hbm as *mut c_void,
-        size_of::<BITMAP>() as i32,
-        (&mut bitmap) as *mut BITMAP as *mut c_void
-    );
-    // println!("bitmap width: {}", bitmap.bmWidth);
-    // println!("bitmap height: {}", bitmap.bmHeight);
-    // println!("bitmap bits pixel: {}", bitmap.bmBitsPixel);
-
-    let mut bi: BITMAPINFOHEADER = BITMAPINFOHEADER {
-        biSize: size_of::<BITMAPINFOHEADER>() as u32,
-        biWidth: bitmap.bmWidth,
-        biHeight: bitmap.bmHeight,
-        biPlanes: 1,
-        biBitCount: 32,
-        biCompression: BI_RGB,
-        biSizeImage: 0,
-        biXPelsPerMeter: 0,
-        biYPelsPerMeter: 0,
-        biClrUsed: 0,
-        biClrImportant: 0,
-    };
-
-    let bitmap_size: usize = (((bitmap.bmWidth * 32 + 31) / 32) * 4 * bitmap.bmHeight) as usize;
-    // println!("bitmap size: {}", bitmap_size);
-    // let mut buffer: Vec<u8> = vec![0; bitmap_size];
-
-    // let h_dib = GlobalAlloc(GHND, bitmap_size);
-    // let lpbitmap = GlobalLock(h_dib);
-    // println!("bitmap {:p}", lpbitmap);
-    let mut buffer: Vec<u8> = vec![0; bitmap_size];
-
-    GetDIBits(
-        dc_window,
-        hbm,
-        0,
-        bitmap.bmHeight as u32,
-        // lpbitmap,
-        buffer.as_mut_ptr() as *mut c_void,
-        (&mut bi) as *mut BITMAPINFOHEADER as *mut BITMAPINFO,
-        DIB_RGB_COLORS
-    );
-
-    // let buffer: Vec<u8> = Vec::from_raw_parts(lpbitmap as *mut u8, bitmap_size, bitmap_size);
-    // for i in 0..10 {
-    //     println!("{}", buffer[i]);
-    // }
-
-    // println!("{}", buffer[0]);
-
-    DeleteObject(hbm as *mut c_void);
-    DeleteObject(dc_mem as *mut c_void);
-    ReleaseDC(null_mut(), dc_window);
-
     Ok(buffer)
 }
 
-#[cfg(windows)]
-pub fn capture_absolute(rect: &PixelRect) -> Result<Vec<u8>, String> {
-    unsafe {
-        unsafe_capture(&rect)
-    }
-}
-
-#[cfg(windows)]
-pub fn capture_absolute_image(rect: &PixelRect) -> Result<image::RgbImage, String> {
-    let raw: Vec<u8> = match capture_absolute(rect) {
-        Err(s) => {
-            return Err(s);
-        },
-        Ok(v) => v,
-    };
-
-    let height = rect.height as u32;
-    let width = rect.width as u32;
-
-    let mut img = ImageBuffer::from_fn(
+pub fn capture_absolute_image(
+    PixelRect {
+        left,
+        top,
         width,
         height,
-        move |x, y| {
-            let y = height - y - 1;
-            let b = raw[((y * width + x) * 4 + 0) as usize];
-            let g = raw[((y * width + x) * 4 + 1) as usize];
-            let r = raw[((y * width + x) * 4 + 2) as usize];
-            image::Rgb([r, g, b])
+    }: &PixelRect,
+) -> Result<image::RgbImage, String> {
+    // simply use the first screen.
+    // todo: multi-screen support
+    let screen = screenshots::Screen::all().ok_or("cannot get DisplayInfo")?[0];
+    let (buffer, is_bgra) = screen
+        .capture_area(*left, *top, *width as u32, *height as u32)
+        .ok_or("capture failed")?;
+    Ok(RgbImage::from_fn(*width as u32, *height as u32, |x, y| {
+        let offset = (y * (*width as u32) + x) as usize;
+        if is_bgra {
+            Rgb([buffer[offset + 2], buffer[offset + 1], buffer[offset]])
+        } else {
+            Rgb([buffer[offset], buffer[offset + 1], buffer[offset + 2]])
         }
-    );
-
-    Ok(img)
+    }))
 }
 
-#[cfg(windows)]
 pub fn get_color(x: u32, y: u32) -> Color {
     let im = capture_absolute(&PixelRect {
         left: x as i32,
         top: y as i32,
         width: 1,
         height: 1,
-    }).unwrap();
-
-    let b = im[0];
-    let g = im[1];
-    let r = im[2];
-    Color(r, g, b)
-}
+    })
+    .unwrap();
+    Color::from(im[2], im[1], im[0])
+}

src/common/buffer.rs

@@ -3,7 +3,7 @@ pub struct Buffer {
 }
 
 impl Buffer {
-    fn new(width: usize, height: usize) -> Buffer {
+    fn new_zeroed(width: usize, height: usize) -> Buffer {
         Buffer {
             data: vec![vec![0.0; width]; height]
         }

src/common/mod.rs

@@ -141,7 +141,7 @@ impl RawCaptureImage {
         for i in rect.left..rect.left + rect.width {
             for j in rect.top..rect.top + rect.height {
                 let x = i;
-                let y = self.h as i32 - j - 1;
+                let y = j;
                 let b: u8 = self.data[((y * self.w as i32 + x) * 4) as usize];
                 let g: u8 = self.data[((y * self.w as i32 + x) * 4 + 1) as usize];
                 let r: u8 = self.data[((y * self.w as i32 + x) * 4 + 2) as usize];