【图形图像】Android SurfaceFlinger之NativeWindow

1、EGLNativeWindowType

Android的GUI构建于OpenGL ES，对于2D图形来说还可以使用Skia库（https://skia.org/）。在OpenGL ES与底层的framebuffer之间，还有一层Native介质，如NativeWindow、NativeDisplay和NativePixmap，这里讨论NativeWindow。OpenGL ES是跨平台的，为了适配不同的运行环境，需要EGL将其Native化，如下所示的eglCreateWindowSurface。

EGLAPI EGLSurface EGLAPIENTRY eglCreateWindowSurface(EGLDisplay dpy, EGLConfig config,
                  EGLNativeWindowType win,
                  const EGLint *attrib_li

eglCreateWindowSurface中的EGLNativeWindowType就是个平台相关的类型，如下所示：

#if defined(_WIN32) || defined(__VC32__) && !defined(__CYGWIN__) && !defined(__SCITECH_SNAP__) /* Win32 and WinCE */
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN 1
#endif
#include <windows.h>

typedef HDC     EGLNativeDisplayType;
typedef HBITMAP EGLNativePixmapType;
typedef HWND    EGLNativeWindowType;

#elif defined(__WINSCW__) || defined(__SYMBIAN32__)  /* Symbian */

typedef int   EGLNativeDisplayType;
typedef void *EGLNativeWindowType;
typedef void *EGLNativePixmapType;

#elif defined(__ANDROID__) || defined(ANDROID)

#include <android/native_window.h>

struct egl_native_pixmap_t;

typedef struct ANativeWindow*           EGLNativeWindowType;
typedef struct egl_native_pixmap_t*     EGLNativePixmapType;
typedef void*                           EGLNativeDisplayType;

#elif defined(__unix__)

/* X11 (tentative)  */
#include <X11/Xlib.h>
#include <X11/Xutil.h>

typedef Display *EGLNativeDisplayType;
typedef Pixmap   EGLNativePixmapType;
typedef Window   EGLNativeWindowType;

#else
#error "Platform not recognized"
#endif

/* EGL 1.2 types, renamed for consistency in EGL 1.3 */
typedef EGLNativeDisplayType NativeDisplayType;
typedef EGLNativePixmapType  NativePixmapType;
typedef EGLNativeWindowType  NativeWindowType;

可以看出，在Android系统上，EGLNativeWindowType为指向ANativeWindow结构的指针， 如下所示：

struct ANativeWindow
{
#ifdef __cplusplus
    ANativeWindow()
        : flags(0), minSwapInterval(0), maxSwapInterval(0), xdpi(0), ydpi(0)
    {
        common.magic = ANDROID_NATIVE_WINDOW_MAGIC;
        common.version = sizeof(ANativeWindow);
        memset(common.reserved, 0, sizeof(common.reserved));
    }
    void incStrong(const void* id) const {
        common.incRef(const_cast<android_native_base_t*>(&common));
    }
    void decStrong(const void* id) const {
        common.decRef(const_cast<android_native_base_t*>(&common));
    }
#endif

    struct android_native_base_t common;

    uint32_t flags;
    int   minSwapInterval;
    int   maxSwapInterval;
    float xdpi;
    float ydpi;
    intptr_t    oem[4];

    int     (*setSwapInterval)(struct ANativeWindow* window,
                int interval);
    int     (*dequeueBuffer)(struct ANativeWindow* window,
                struct ANativeWindowBuffer** buffer);
    int     (*lockBuffer)(struct ANativeWindow* window,
                struct ANativeWindowBuffer* buffer);
    int     (*queueBuffer)(struct ANativeWindow* window,
                struct ANativeWindowBuffer* buffer);
    int     (*query)(const struct ANativeWindow* window,
                int what, int* value);
    int     (*perform)(struct ANativeWindow* window,
                int operation, ... );
    int     (*cancelBuffer)(struct ANativeWindow* window,
                struct ANativeWindowBuffer* buffer);

    void* reserved_proc[2];
};

2、FramebufferNativeWindow

在老式的Android版本中有个专门的FramebufferNativeWindow用来实现ANativeWindow的功能，如下所示：

class FramebufferNativeWindow 
    : public EGLNativeBase<
        ANativeWindow, 
        FramebufferNativeWindow, 
        LightRefBase<FramebufferNativeWindow> >

不过在最新的Android系统（Android 7）中，FramebufferNativeWindow已被移除，看到Android源码中有如下一条提交记录：

libui: Remove FramebufferNativeWindow
We no longer support the framebuffer device, so FramebufferNativeWindow is no longer relevant.

FramebufferNativeWindow是属于libui库的，虽被移除，但其实现逻辑值得借鉴。在FramebufferNativeWindow的构造函数中，首先打开gralloc模块，即dlopen相关的gralloc库，然后打开fb0和gpu0设备，再分配需要的buffer（用于framebuffer device），这个buffer即NativeBuffer的数据结构如下：

class NativeBuffer 
    : public EGLNativeBase<
        ANativeWindowBuffer, 
        NativeBuffer, 
        LightRefBase<NativeBuffer> >

typedef struct ANativeWindowBuffer
{
#ifdef __cplusplus
    ANativeWindowBuffer() {
        common.magic = ANDROID_NATIVE_BUFFER_MAGIC;
        common.version = sizeof(ANativeWindowBuffer);
        memset(common.reserved, 0, sizeof(common.reserved));
    }
    void incStrong(const void* id) const {
        common.incRef(const_cast<android_native_base_t*>(&common));
    }
    void decStrong(const void* id) const {
        common.decRef(const_cast<android_native_base_t*>(&common));
    }
#endif

    struct android_native_base_t common;
    int width;
    int height;
    int stride;
    int format;
    int usage;
    void* reserved[2];
    buffer_handle_t handle;
    void* reserved_proc[8];
} ANativeWindowBuffer_t;

最后是一些属性和接口赋值操作，其中最重要的一点是多缓冲技术，至少2个buffer，可以理解为一个screen-buffer（用户直接看到的东西）和一个off-screen-buffer（即将呈现给用户的东西），这就需要所谓的swap buffer，因为在屏幕上绘制东西需要时间，而屏幕刷新有一定的帧率（fps），为了保证画面流畅，就要在off-screen-buffer准备好数据后通过swap buffer把数据发送到screen-buffer以显示。举一个例子，对于单个buffer来说，如果绘制一个画面需要10秒，而屏幕每隔1秒刷新一次，用户就会感觉很卡，前9秒看到的都是不同的且不完整的画面。在NativeWindow中有两个重要的接口dequeueBuffer和queueBuffer，EGL通过dequeueBuffer来申请一个buffer，当EGL对一块buffer渲染完成后，queueBuffer用来unlock和post buffer，其中用到了条件变量用于同步操作。

3、Surface

另一个本地窗口是Surface，针对应用程序，同样继承自ANativeWindow：

class Surface
    : public ANativeObjectBase<ANativeWindow, Surface, RefBase>

Surface属于libgui库的东西，承担着应用进程中的UI显示需求，其分配的内存空间不属于帧缓冲区，系统中所有应用程序绘制的图像数据由SurfaceFlinger收集。Surface有两个重要的成员变量，mGraphicBufferProducer和mSlots，前者用来获取buffer，为buffer的生产者，后者则记录这些缓冲区，NUM_BUFFER_SLOTS为64。

sp<IGraphicBufferProducer> mGraphicBufferProducer;
BufferSlot mSlots[NUM_BUFFER_SLOTS];

struct BufferSlot {
        sp<GraphicBuffer> buffer;
        Region dirtyRegion;
    };

在Surface的构造函数中同样是给其成员变量赋值，做一些初始化工作，比如说如下的dequeueBuffer：

ANativeWindow::dequeueBuffer    = hook_dequeueBuffer;

int Surface::hook_dequeueBuffer(ANativeWindow* window,
        ANativeWindowBuffer** buffer, int* fenceFd) {
    Surface* c = getSelf(window);
    return c->dequeueBuffer(buffer, fenceFd);
}

int Surface::dequeueBuffer(android_native_buffer_t** buffer, int* fenceFd) {
    ATRACE_CALL();
    ALOGV("Surface::dequeueBuffer");

    uint32_t reqWidth;
    uint32_t reqHeight;
    PixelFormat reqFormat;
    uint32_t reqUsage;

    {
        Mutex::Autolock lock(mMutex);

        reqWidth = mReqWidth ? mReqWidth : mUserWidth;
        reqHeight = mReqHeight ? mReqHeight : mUserHeight;

        reqFormat = mReqFormat;
        reqUsage = mReqUsage;

        if (mSharedBufferMode && mAutoRefresh && mSharedBufferSlot !=
                BufferItem::INVALID_BUFFER_SLOT) {
            sp<GraphicBuffer>& gbuf(mSlots[mSharedBufferSlot].buffer);
            if (gbuf != NULL) {
                *buffer = gbuf.get();
                *fenceFd = -1;
                return OK;
            }
        }
    } // Drop the lock so that we can still touch the Surface while blocking in IGBP::dequeueBuffer

    int buf = -1;
    sp<Fence> fence;
    nsecs_t now = systemTime();
    status_t result = mGraphicBufferProducer->dequeueBuffer(&buf, &fence,
            reqWidth, reqHeight, reqFormat, reqUsage);
    mLastDequeueDuration = systemTime() - now;

    if (result < 0) {
        ALOGV("dequeueBuffer: IGraphicBufferProducer::dequeueBuffer"
                "(%d, %d, %d, %d) failed: %d", reqWidth, reqHeight, reqFormat,
                reqUsage, result);
        return result;
    }

    Mutex::Autolock lock(mMutex);

    sp<GraphicBuffer>& gbuf(mSlots[buf].buffer);

    // this should never happen
    ALOGE_IF(fence == NULL, "Surface::dequeueBuffer: received null Fence! buf=%d", buf);

    if (result & IGraphicBufferProducer::RELEASE_ALL_BUFFERS) {
        freeAllBuffers();
    }

    if ((result & IGraphicBufferProducer::BUFFER_NEEDS_REALLOCATION) || gbuf == 0) {
        result = mGraphicBufferProducer->requestBuffer(buf, &gbuf);
        if (result != NO_ERROR) {
            ALOGE("dequeueBuffer: IGraphicBufferProducer::requestBuffer failed: %d", result);
            mGraphicBufferProducer->cancelBuffer(buf, fence);
            return result;
        }
    }

    if (fence->isValid()) {
        *fenceFd = fence->dup();
        if (*fenceFd == -1) {
            ALOGE("dequeueBuffer: error duping fence: %d", errno);
            // dup() should never fail; something is badly wrong. Soldier on
            // and hope for the best; the worst that should happen is some
            // visible corruption that lasts until the next frame.
        }
    } else {
        *fenceFd = -1;
    }

    *buffer = gbuf.get();

    if (mSharedBufferMode && mAutoRefresh) {
        mSharedBufferSlot = buf;
        mSharedBufferHasBeenQueued = false;
    } else if (mSharedBufferSlot == buf) {
        mSharedBufferSlot = BufferItem::INVALID_BUFFER_SLOT;
        mSharedBufferHasBeenQueued = false;
    }

    return OK;
}

dequeueBuffer最终通过Surface的成员函数dequeueBuffer实现，关键操作由其成员变量mGraphicBufferProducer这个buffer生产者来完成，既然有生产者就有消费者，消费者为SurfaceFlinger（libsurfaceflinger中的东西）。Surface由SurfaceControl创建、管理，如下所示：

sp<Surface> SurfaceControl::getSurface() const
{
    Mutex::Autolock _l(mLock);
    if (mSurfaceData == 0) {
        // This surface is always consumed by SurfaceFlinger, so the
        // producerControlledByApp value doesn't matter; using false.
        mSurfaceData = new Surface(mGraphicBufferProducer, false);
    }
    return mSurfaceData;
}

SurfaceControl由Surface合成的客户端SurfaceComposerClient创建，如下所示：

sp<SurfaceControl> SurfaceComposerClient::createSurface(
        const String8& name,
        uint32_t w,
        uint32_t h,
        PixelFormat format,
        uint32_t flags)
{
    sp<SurfaceControl> sur;
    if (mStatus == NO_ERROR) {
        sp<IBinder> handle;
        sp<IGraphicBufferProducer> gbp;
        status_t err = mClient->createSurface(name, w, h, format, flags,
                &handle, &gbp);
        ALOGE_IF(err, "SurfaceComposerClient::createSurface error %s", strerror(-err));
        if (err == NO_ERROR) {
            sur = new SurfaceControl(this, handle, gbp);
        }
    }
    return sur;
}

在SurfaceComposerClient的createSurface中，通过mClient（类型为sp）来createSurface，其中有两个关键变量handle（IBinder）和gbp（IGraphicBufferProducer），mClient在onFirstRef函数中获取与SurfaceFlinger（服务端）的连接，如下代码：

void SurfaceComposerClient::onFirstRef() {
    sp<ISurfaceComposer> sm(ComposerService::getComposerService());
    if (sm != 0) {
        sp<ISurfaceComposerClient> conn = sm->createConnection();
        if (conn != 0) {
            mClient = conn;
            mStatus = NO_ERROR;
        }
    }
}

这样，一个普通的Surface就与SurfaceFlinger建立了联系。由此可见，ISurfaceComposerClient是由ISurfaceComposer::createConnection生成的，在这一过程中，总共涉及了三个匿名的Binder服务，分别为ISurfaceComposer、ISurfaceComposerClient和IGraphicBufferProducer，它们是紧紧相扣的，只能按上面的顺序访问。这些匿名Binder由实名Binder即在ServiceManager中注册过的SurfaceFlinger提供，从下面的函数中可以看出：

/*static*/ sp<ISurfaceComposer> ComposerService::getComposerService() {
    ComposerService& instance = ComposerService::getInstance();
    Mutex::Autolock _l(instance.mLock);
    if (instance.mComposerService == NULL) {
        ComposerService::getInstance().connectLocked();
        assert(instance.mComposerService != NULL);
        ALOGD("ComposerService reconnected");
    }
    return instance.mComposerService;
}

void ComposerService::connectLocked() {
    const String16 name("SurfaceFlinger"); // 轮询查找SurfaceFlinger服务
    while (getService(name, &mComposerService) != NO_ERROR) {
        usleep(250000);
    }
    assert(mComposerService != NULL);

    // Create the death listener.
    class DeathObserver : public IBinder::DeathRecipient {
        ComposerService& mComposerService;
        virtual void binderDied(const wp<IBinder>& who) {
            ALOGW("ComposerService remote (surfaceflinger) died [%p]",
                  who.unsafe_get());
            mComposerService.composerServiceDied();
        }
     public:
        explicit DeathObserver(ComposerService& mgr) : mComposerService(mgr) { }
    };

    mDeathObserver = new DeathObserver(*const_cast<ComposerService*>(this));
    IInterface::asBinder(mComposerService)->linkToDeath(mDeathObserver);
}

前面介绍了Android的GUI系统中两个重要的本地窗口，FramebuferNativeWindow和Surface，前者是专门为SurfaceFlinger服务的，由Gralloc提供支持，后者为应用程序服务，但也由SurfaceFlinger统一管理。