c++ fast screenshots in linux for use with opencv

Question

I'm trying to find a fast way to take screenshots (as in 30 fps or over) for use with opencv in c++

All the information I've found online either involved windows.h or were too slow.

Could someone provide me with some code that achieves this or at least point me in the right direction so I can solve this my self?

Solution 1

You can use this to get the screenshot into a structure of raw pixels. Pass that to OpenCV along with the Width & Height & BitsPerPixel and you should be good.

#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <cstdint>
#include <cstring>
#include <vector>

void ImageFromDisplay(std::vector<uint8_t>& Pixels, int& Width, int& Height, int& BitsPerPixel)
{
    Display* display = XOpenDisplay(nullptr);
    Window root = DefaultRootWindow(display);

    XWindowAttributes attributes = {0};
    XGetWindowAttributes(display, root, &attributes);

    Width = attributes.width;
    Height = attributes.height;

    XImage* img = XGetImage(display, root, 0, 0 , Width, Height, AllPlanes, ZPixmap);
    BitsPerPixel = img->bits_per_pixel;
    Pixels.resize(Width * Height * 4);

    memcpy(&Pixels[0], img->data, Pixels.size());

    XDestroyImage(img);
    XCloseDisplay(display);
}

Then to use it with OpenCV, you can do:

int main()
{
    int Width = 0;
    int Height = 0;
    int Bpp = 0;
    std::vector<std::uint8_t> Pixels;

    ImageFromDisplay(Pixels, Width, Height, Bpp);

    if (Width && Height)
    {
        Mat img = Mat(Height, Width, Bpp > 24 ? CV_8UC4 : CV_8UC3, &Pixels[0]); //Mat(Size(Height, Width), Bpp > 24 ? CV_8UC4 : CV_8UC3, &Pixels[0]); 

        namedWindow("WindowTitle", WINDOW_AUTOSIZE);
        imshow("Display window", img);

        waitKey(0);
    }
    return 0;
}

Solution 2

Based on @abc's answer I came up with the following, using MIT's shared memory extension for X.

@abc's example runs at 120-180 fps and uses ~40% of a Titan X (Maxwell). The following runs at 15000 fps and uses 80% of a Titan X. (Assuming you don't sync. If you do sync, then the framerate drops to @abc's framerate!)

Comment out the break; statement inside the inner loop too run it nonstop.

Version using @abc's class:

// g++ screena.cpp -o screena -lX11 -lXext -Ofast -mfpmath=both -march=native -m64 -funroll-loops -mavx2 `pkg-config opencv --cflags --libs` && ./screena

#include <X11/Xlib.h>
#include <X11/Xutil.h>

#include <X11/extensions/XShm.h>
#include <sys/ipc.h>
#include <sys/shm.h>

#include <opencv2/opencv.hpp>  // This includes most headers!

#include <time.h>
#define FPS(start) (CLOCKS_PER_SEC / (clock()-start))


struct ScreenShot{
    ScreenShot(uint x, uint y, uint width, uint height):
               x(x), y(y), width(width), height(height){

        display = XOpenDisplay(nullptr);
        root = DefaultRootWindow(display);

        XGetWindowAttributes(display, root, &window_attributes);
        screen = window_attributes.screen;
        ximg = XShmCreateImage(display, DefaultVisualOfScreen(screen), DefaultDepthOfScreen(screen), ZPixmap, NULL, &shminfo, width, height);

        shminfo.shmid = shmget(IPC_PRIVATE, ximg->bytes_per_line * ximg->height, IPC_CREAT|0777);
        shminfo.shmaddr = ximg->data = (char*)shmat(shminfo.shmid, 0, 0);
        shminfo.readOnly = False;
        if(shminfo.shmid < 0)
            puts("Fatal shminfo error!");;
        Status s1 = XShmAttach(display, &shminfo);
        printf("XShmAttach() %s\n", s1 ? "success!" : "failure!");

        init = true;
    }

    void operator() (cv::Mat& cv_img){
        if(init)
            init = false;

        XShmGetImage(display, root, ximg, 0, 0, 0x00ffffff);
        cv_img = cv::Mat(height, width, CV_8UC4, ximg->data);
    }

    ~ScreenShot(){
        if(!init)
            XDestroyImage(ximg);

        XShmDetach(display, &shminfo);
        shmdt(shminfo.shmaddr);
        XCloseDisplay(display);
    }

    Display* display;
    Window root;
    XWindowAttributes window_attributes;
    Screen* screen;
    XImage* ximg;
    XShmSegmentInfo shminfo;

    int x, y, width, height;

    bool init;
};


int main(){
    ScreenShot screen(0, 0, 1920, 1080);
    cv::Mat img;

    for(uint i;; ++i){
        double start = clock();

        screen(img);

        if(!(i & 0b111111))
            printf("fps %4.f  spf %.4f\n", FPS(start), 1 / FPS(start));
            break;

    }

    cv::imshow("img", img);
    cv::waitKey(0);
}

"Naked" version:

// g++ xshm2.c -o xshm2 -lX11 -lXext `$cv`-Ofast -mfpmath=both -march=native -m64 -funroll-loops -mavx2 && ./xshm2

#include <X11/Xlib.h>
#include <X11/Xutil.h>

#include <X11/extensions/XShm.h>
#include <sys/ipc.h>
#include <sys/shm.h>

#include <opencv2/opencv.hpp>  // This includes most headers!

#include <time.h>
#define FPS(start) (CLOCKS_PER_SEC / (clock()-start))

// Using one monitor DOESN'T improve performance! Querying a smaller subset of the screen DOES
const uint WIDTH  = 1920>>0;
const uint HEIGHT = 1080>>0;

// -------------------------------------------------------
int main(){
    Display* display = XOpenDisplay(NULL);
    Window root = DefaultRootWindow(display);  // Macro to return the root window! It's a simple uint32
    XWindowAttributes window_attributes;
    XGetWindowAttributes(display, root, &window_attributes);
    Screen* screen = window_attributes.screen;
    XShmSegmentInfo shminfo;
    XImage* ximg = XShmCreateImage(display, DefaultVisualOfScreen(screen), DefaultDepthOfScreen(screen), ZPixmap, NULL, &shminfo, WIDTH, HEIGHT);

    shminfo.shmid = shmget(IPC_PRIVATE, ximg->bytes_per_line * ximg->height, IPC_CREAT|0777);
    shminfo.shmaddr = ximg->data = (char*)shmat(shminfo.shmid, 0, 0);
    shminfo.readOnly = False;
    if(shminfo.shmid < 0)
        puts("Fatal shminfo error!");;
    Status s1 = XShmAttach(display, &shminfo);
    printf("XShmAttach() %s\n", s1 ? "success!" : "failure!");

    cv::Mat img;

    for(int i; ; i++){
        double start = clock();

        XShmGetImage(display, root, ximg, 0, 0, 0x00ffffff);
        img = cv::Mat(HEIGHT, WIDTH, CV_8UC4, ximg->data);

        if(!(i & 0b111111))
            printf("fps %4.f  spf %.4f\n", FPS(start), 1 / FPS(start));
        break;
    }

    cv::imshow("img", img);
    cv::waitKey(0);

    XShmDetach(display, &shminfo);
    XDestroyImage(ximg);
    shmdt(shminfo.shmaddr);
    XCloseDisplay(display);
    puts("Exit success!");
}

Solution 3

I made a faster functor based screenshot code using Brandon's answer:

#include <opencv2/opencv.hpp>
#include <X11/Xlib.h>
#include <X11/Xutil.h>

class ScreenShot
{
    Display* display;
    Window root;
    int x,y,width,height;
    XImage* img{nullptr};
public:
    ScreenShot(int x, int y, int width, int height):
        x(x),
        y(y),
        width(width),
        height(height)
    {
        display = XOpenDisplay(nullptr);
        root = DefaultRootWindow(display);
    }

    void operator() (cv::Mat& cvImg)
    {
        if(img != nullptr)
            XDestroyImage(img);
        img = XGetImage(display, root, x, y, width, height, AllPlanes, ZPixmap);
        cvImg = cv::Mat(height, width, CV_8UC4, img->data);
    }

    ~ScreenShot()
    {
        if(img != nullptr)
            XDestroyImage(img);
        XCloseDisplay(display);
    }
};

And here is how you would use it in a loop (exit by pressing q):

int main()
{
    ScreenShot screen(0,0,1920,1080);
    cv::Mat img;

    while(true) 
    {
        screen(img);

        cv::imshow("img", img);
        char k = cv::waitKey(1);
        if (k == 'q')
            break;
    }
}

This is about 39% faster on my machine.

Solution 4

#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>
#include <cstdint>
#include <cstring>
#include <vector>

using namespace cv;

struct ScreenShot
{
    ScreenShot(int x, int y, int width, int height):
        x(x),
        y(y),
        width(width),
        height(height)
    {
        display = XOpenDisplay(nullptr);
        root = DefaultRootWindow(display);

        init = true;
    }

    void operator() (Mat& cvImg)
    {
        if(init == true)
            init = false;
        else
            XDestroyImage(img);

        img = XGetImage(display, root, x, y, width, height, AllPlanes, ZPixmap);

        cvImg = Mat(height, width, CV_8UC4, img->data);
    }

    ~ScreenShot()
    {
        if(init == false)
            XDestroyImage(img);

        XCloseDisplay(display);
    }

    Display* display;
    Window root;
    int x,y,width,height;
    XImage* img;

    bool init;
};

int main(int, char**)
{
    for(;;){
        ScreenShot screen(0,0,1366,768);

        Mat img;
        screen(img);

        imshow("img", img);
        if(waitKey(30) >= 0) break;
    }
    return 0;
}

I used the following code, based in the answer posted by @abc.

In addition, I was using Visual Studio Code and g++ to compile:

g ++ -std=c++0x main.cpp -lX11 'pkg-config --cflags opencv' 'pkg-config --libs opencv' -o main

The path for Xlib.h and Xutil.h (ubuntu 14.04): /usr/include/X11

View more solutions

Comments

[Brandon]

Oh and to speed it up more, you can replace the memcpy with a call to the OpenCV api for images from pixels or w/e you need the pixel for..

[user237330]

Could you also provide an example of how to use this with opencv? (like displaying the output image using imshow()). I'm having a bit of trouble.

[Brandon]

Ok. The above was edited so there is now an example of using it with OpenCV. I originally thought you already knew how to use OpenCV so I had left that out. Another thing is that instead of doing ImageFromDisplay(Pixels,.... You could make it take an OpenCV image directly for more speed.

[user237330]

I used this to compile the above code: g++ main.cpp -std=c++0x -o app 'pkg-config --cflags --libs opencv' It compiled successfully, except when i try to run the compiled program, it results in a segmentation fault. Does anybody have an idea why?

[edi9999]

you also need to include the libs for x11 , eg g++ main.cpp -std=c++0x -o app 'pkg-config --cflags --libs opencv x11'

[abc]

This leaks memory. Use XDestroyImage(img); instead of XFree(img); in the function ImageFromDisplay.

[étale-cohomology]

This is actually 3x faster on my pc! Thanks for sharing

[abc]

Hmm... I'm not really sure, but should clock() be used here? I'm having doubts because CLOCKS_PER_SEC/clock() returns the CPU time and not wall time (15000 fps seems insanely large). Though when I used the chrono c++ library to measure performance your method was faster, so +1 from me.

[étale-cohomology]

@abc I have no idea about clock! You may be right :) How much faster was it?

[abc]

I got about 50 fps on my machine using my code and 120/140 fps with your code.

[Slug Pue]

@user237330 I was also getting segmentation faults. This was fixed by instead of first copying from the Ximage to Pixels and then Pixels to Mat, directly copying Ximage to Mat

[NMO]

Additionally, you need to add #include "opencv2/core/core.hpp" #include "opencv2/highgui.hpp" and using namespace cv;

[Snackoverflow]

I know the question is about Linux but would this work on Windows? If not, is there a similar method that can be used for Windows to achieve similar efficiency?

[étale-cohomology]

This code snippet uses (Xlib)[x.org/releases/X11R7.7/doc/libX11/libX11/libX11.html]‌​, which is a library to interact with the (X Window System)[en.wikipedia.org/wiki/X_Window_System]. I don't know if Windows can run the X Window System, and, if it does, I don't know if it uses Xlib. But Windows must certainly have its own efficient APIs to get screen pixels...

[Nicholas Pipitone]

@anddero DXGI's AcquireNextFrame does a similar thing, but it takes a million years to setup the API calls correctly lol

[Thenujan Sandramohan]

Had to use the following to compile g++ $(pkg-config opencv4 --cflags) -std=c++0x main.cpp -lX11 `pkg-config --cflags opencv4` `pkg-config --libs opencv4` -o main