[WebRTC] 안드로이드 I420 Buffer관련 클래스

WebRTC 에서는 VideoFrame을 위해 TextureBuffer와 I420Buffer를 사용.
I420Buffer는 Buffer 클래스를 상속받은 클래스이며, Buffer 는 네이티브(jni) 라이브러리에게 버퍼의 크기나 포인터를 알려주는 역할이며, I420Buffer는 YUV 포인터의 위치를 전달하는 역할을 하게된다.

 

TextureBuffer

일반 rgb bitmap 이미지를 VideoFrame으로 변경하려면 대략 아래처럼 처리가 이루어진다.

// 텍스처 생성
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);


// TextureBuffer 생성
Matrix transform = new Matrix();
YuvConverter yuvConverter = new YuvConverter();
TextureBufferImpl buffer = new TextureBufferImpl(width, height, 
                                                 VideoFrame.TextureBuffer.Type.RGB, 
                                                 textures[0],
                                                 transform,
                                                 textureHelper.getHandler(),
                                                 yuvConverter, 
                                                 null);
                                                 
// 텍스처에 특정 Bitmap bitmap 이미지 로드
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILGER, GLES20.GL_NEAREST);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0);
    
    
// 텍스처버퍼를 i420 버퍼로 변경
VideoFrame.I420Buffer i420buffer = buffer.toI420(); // yuvConverter.convert(buffer);
    
// 비디오 프레임 생성
VideoFrame videoFrame = new VideoFrame(i420buffer, 0, timestamp);

    

 

위와 같이 rgb 포맷의 경우 i420 변환이 이루어지게 되며, 이때 yuvConverter의 렌더링 로직을 통해 opengl 렌더링으로 변환이 이루어진다. 이미 i420포맷으로 전달되는 데이터의 경우 변환작업이 필요 없어 지므로, 해당 데이터에서 직접 i420 버퍼를 생성하게 된다.

안드로이드 카메라나 외부 모듈이 OpenGLES 의 프레임버퍼(FBO)의 텍스처에 렌더링 한 경우 i420 포맷(일반적인 경우)로 저장된 경우, 이미 420 포맷의 버퍼를 가지고 있으므로 렌더링(i420변환) 없이 바로 I420Buffer를 생성 할 수 있도록 제공되는 함수를 먼저 살펴본다. 
JavaI420Buffer 클래스는 VideoFrame.I420Buffer를 상속받은 클래스이며, I420 버퍼 생성이나 변형등의 메쏘드를 제공한다. 

wrap

public static JavaI420Buffer wrap(int width, int height, 
                                  ByteBuffer dataY, int strideY, 
                                  ByteBuffer dataU, int strideU, 
                                  ByteBuffer dataV, int strideV, 
                                  Runnnable releaseCallback) {
    dataY = dataY.slice();
    dataU = dataU.slice();
    dataV = dataV.slice();
    
    final int chromaWidth = (width + 1) / 2;
    final int chromaHeight = (height + 1) / 2;
    
    final int capacityY = strideY * (height - 1) + width;
    if( dataY.capacity() < capacityY ) {
        throw new IllegalArgumentException("");
    }
    
    final int capacityU = strideU * (chromaHeight - 1) + chromaWidth;
    if( dataU.capacity() < capacityU ) {
        throw new IllegalArgumentException("");
    }
    
    final innt capacityV = strideV * (chromaHeight - 1) + chromaWidth;
    if( dataV.capacity() < capacityV ) {
        throw new IllegalArgumentException("");
    }
    
    return new JavaI420Buffer( width, height,
                               dataY, strideY,
                               dataU, strideU,
                               dataV, strideV,
                               releaseCallback);
}

 

allocate

public static JavaI420Buffer allocate(int width, int height) {
    int chromaHeight = (height + 1) / 2;
    int strideUV = (width + 1) / 2;
    int yPos = 0;
    int uPos = yPos + width * height;
    int vPos = uPos + strideUV * chromaHeight;
    
    ByteBuffer buffer = JniCommon.nativeAllocateByteBuffer(width * height + 2 * strideUV * chromaHeight);
    buffer.position(yPos);
    buffer.limit(uPos);
    ByteBuffer dataY = buffer.slice();
    
    buffer.position(uPos);
    buffer.limit(uPos);
    ByteBuffer dataU = buffer.slice();
    
    buffer.position(vPos);
    buffer.limit(vPos + strideUV * chromaHeight);
    ByteBuffer dataV = buffer.slice();
    
    return new JavaI420Buffer(width, height, 
                              dataY, width, 
                              dataU, strideUV, 
                              dataV, strideUV, 
                              ()->{ JniCommon.nativeFreeByteBuffer(buffer); });
}

 

 

crop

public static VideoFrame.Buffer cropAndScaleI420(final I420Buffer buffer, 
                                                 int cropX, int cropY,
                                                 int cropWidth, int cropHeight,
                                                 int scaleWidth, int scaleHeight) {
    if( cropWidth == scaleWidth && cropHeight == scaleHeight ) {
        ByteBuffer dataY = buffer.getDataY();
        ByteBuffer dataU = buffer.getDataU();
        ByteBuffer dataV = buffer.getDataV();
        
        dataY.position(cropX + cropY * buffer.getStrideY());
        dataU.position(cropX / 2 + cropY / 2 * buffer.getStrideU());
        dataV.position(cropX / 2 + cropY / 2 * buffer.getStrideV());
        
        buffer.retain();
        return JavaI420Buffer.wrap(scaleWidth, scaleHeight, 
                                   dataY.slice(), buffer.getStrideY(),
                                   dataU.slice(), buffer.getStrideU(),
                                   dataV.slice(), buffer.getStrideV(),
                                   buffer::release);
    }
    
    JavaI420Buffer newBuffer = JavaI420Buffer.allocate(scaleWidth, scaleHeight);
    nativeCropAndScaleI420(buffer.getDataY(), buffer.getStrideY(),
                           buffer.getDataU(), buffer.getStrideU(),
                           buffer.getDataV(), buffer.getStrideV(),
                           cropX, cropY, cropWidth, cropHeight,
                           newBuffer.getDataY(), newBuffer.getStrideY(),
                           newBuffer.getDataU(), newBuffer.getStrideU(),
                           newBuffer.getDataV(), newBuffer.getStrideV(),
                           scaleWidth, scaleHeight);
    return newBuffer;
}

 

이렇게 생성된 JavaI420Buffer 객체는 VideoFrame 생성시 인자로 넣어주면 VideoFrame 객체를 생성할 수 있다.

---

TextureBufferImpl

일반 rgb 텍스처 이미지를 VideoFrame으로 생성하려면 앞서 언급한바와 같이 별도 변환과정을 거쳐야 하는데, 속도를 위해 opengl 렌더링을 사용한다.
텍스처 버퍼는 텍스처 정보와 YuvConverter, RefCountMonitor 를 가진 객체이며, 주요 메쏘드는 다음과 같다.

public VideoFrame.I420Buffer toI420() {
    return ThreadUtils.invokeAtFrontUninterruptibly( toI420Handler, () -> yuvConverter.convert(this));
}

public VideoFrame.Buffer cropAndScale( int cropX, 
                                       int cropY,
                                       int cropWidth,
                                       int cropHeight,
                                       int scaleWidth,
                                       int scaleHeight) {
    // 참고: webrtc 의 경우 y=0 이 상단, opengl y = 하단에서 시작됨
    final Matrix cropAndScaleMatrix = new Matrix();
    final int cropYFromBottom = height - (cropY + cropHeight);
    cropAndScaleMatrix.preTranslate(cropX / (float)width, cropYFromBottom / (float)height);
    cropAndScaleMatrix.preScale(cropWidth / (float)width, cropHeight / (float)height);
    
    
    final Matrix newMatrix = new Matrix(cropAndScaleMatrix);
    final int unscaledWidth = (int)Math.round(unscaledWidth * cropWidth / (float)width);
    final int unscaledHeight = (int)Math.round(unscaledHeight * cropHeight / (float)height);
    
    refCountDelegate.retain();
    return new TextureBufferImpl( unScaledWidth, unscaledHeight, 
                                  scaledWidth, scaledHeight, 
                                  texturePixelType, 
                                  textureId, 
                                  newMatrix, 
                                  toI420Handler, 
                                  yuvConverter, 
                                  null ); 
}
                                       

 

YuvConverter

convert( TextureBuffer inputTextureBuffer )

주요 프로퍼티

private final GlTextureFrameBuffer i420TextureFrameBuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
private final GlGenericDrawer drawer = new GlGenericDrawer( FRAGMENT_SHADER, shaderCallbacks);
private final VideoFrameDrawer videoFrameDrawer = new VideoFrameDrawer();

 

GlTextureFrameBuffe: FBO 생성
GlGenericDrawer: 사각형 영역 렌더링
VideoFrameDrawer: 렌더링을 위한 각 프레임 처리

TextureBuffer preparedBuffer = inputTextureBuffer.retain();
final int frameWidth = inputTextureBuffer.getWidth();
final int frameHeight = inputTetureBuffer.getHeight();
final int stride = ((frameWidth + 7) / 8) * 8;
final int uvHeight = (frameHeight + 1) / 2;
final int totalHeight = frameHeight + uvHeight;
final ByteBuffer i420ByteBuffer = JniCommon.nativeAllocateByteBuffer( stride * totalHeight );
final int viewportWidth = stride / 4;
final Matrix renderMatrix = new Matrix();
renderMatrix.preTranslate(0.5f, 0.5f);
renderMatrix.preScale(1.f, -1f);
renderMatrix.preTranslate(-0.5f, -0.5f);


// 프레임 버퍼 설정
i420TextureFrameBuffer.setSize(viewportWidth, totalHeight);

// 프레임버퍼 바인딩
GLES20.glBindFrameBuffer(GLES20.GL_FRAMEBUFFER, i420TextureFrameBuffer.getFrameBufferId());

private static final float[] yCoeffs = new float[] { 0.256799f, 0.504129f, 0.0979059f, 0.0627451f};
private static final float[] uCoeffs = new float[] { -0.148223f, -0.290993f, 0.439216f, 0.501961f};
private static final float[] vCoeffs = new float[] { 0.439216f, -0.367788f, -0.0714274f, 0.501961f};

// Y 렌더링
float[] coeffs = yCoeffs;
float stepSize = 1.0f;
Matrix fianlMatrix = new Matrix(preparedBuffer.getTransformMatrix());
finalMatrix.preConcat(renderMatrix);
float[] finalGlMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(finalMatrix);
switch(preparedBuffer.getType()) {
    case OES:
    genericDrawer.drawOes(....);
    break;
    
    case RGB:
    genericDrawer.drawRgb(....);
    break;
}

// U 렌더링
coeffs = uCoeffs;
stepSize = 2.0f;
	.
    .
    
// V 렌더링
coeffs = vCoeffs;
stepSize = 2.0f;
    .
    .
    
// bytebuffer에 텍스처의 픽셀 읽기
GLES20.glReadPixels(0, 
                    0, 
                    i420TextureFrameBuffer.getWidth(), 
                    i420TextureFrameBuffer.getHeight(),
                    GLES20.GL_RGBA,
                    GLES20.GL_UNSIGNED_BYTE,
                    i420ByteBuffer);
                    
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);


// bytebuffer를 yuv 단위별 나누기
final int yPos = 0;
final int uPos = yPos + stride * frameHeight;
final int vPos = uPos + stride / 2;

// Y
i420ByteBuffer.position(yPos);
i420ByteBuffer.limit(yPos + stride * frameHeight);
final ByteBuffer dataY = i420ByteBuffer.slice();

// U
i420ByteBuffer.position(uPos);
final int uvSize = stride * (uvHeight - 1) + stride / 2;
i420ByteBuffer.limit(uPos + uvSize);
final ByteBuffer dataU = i420ByteBuffer.slice();

// V
i420ByteBuffer.position(vPos);
i420ByteBuffer.limit(vPos + uvSize);
final ByteBuffer dataV = i420ByteBuffer.slice();

preparedBuffer.release();

return JavaI420Buffer.wrap(frameWidth, frameHeight, 
                           dataY, stride, 
                           dataU, stride, 
                           dataV, stride, 
                           ()-> { JniCommon.nativeFreeByteBuffer(i420ByteBuffer); });

 

 

GlGenericDrawer

public void draw(int[] textureId, float[] texMatrix, 
                    int frameWidth, int frameHeight,
                    int viewportX, int viewportY,
                    int viewportWdith, int viewportHeight ) {
    // vertex shader
    final int shader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
    GLES20.glShaderSource(shader, vertexSource);
    GLES20.glCompileShader(shader);
    int[] compileStatus = new int[] {GLES20.GL_FALSE};
    GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
    if (compileStatus[0] != GLES20.GL_TRUE) {
    	// error
    }
    
    // fragment shader
    final int shader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
    GLES20.glShaderSource(shader, fragmentSource);
    GLES20.glCompileShader(shader);
    int[] compileStatus = new int[] {GLES20.GL_FALSE};
    GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
    if (compileStatus[0] != GLES20.GL_TRUE) {
    	// error
    }
    
    
    // program
    program = GLES20.glCreateProgram();
    GLES20.glAttachShader(program, vertexShader);
    GLES20.glAttachShader(program, fragmentShader);
    GLES20.glLinkProgram(program);
    int[] linkStatus = new int[] {GLES20.GL_FALSE};
    GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
    
    
    // According to the documentation of glLinkProgram():
    // "After the link operation, applications are free to modify attached shader objects, compile
    // attached shader objects, detach shader objects, delete shader objects, and attach additional
    // shader objects. None of these operations affects the information log or the program that is
    // part of the program object."
    // But in practice, detaching shaders from the program seems to break some devices. Deleting the
    // shaders are fine however - it will delete them when they are no longer attached to a program.
    GLES20.glDeleteShader(vertexShader);
    GLES20.glDeleteShader(fragmentShader);
    
    
   
    
    GLES20.glUseProgram(program);
    
    if( shaderType == ShaderType.YUV ) {
        GLES20.glUniform1i( GLES20.glGetUniformLocation(program, "y_tex"), 0);
        GLES20.glUniform1i( GLES20.glGetUniformLocation(program, "u_tex"), 1);
        GLES20.glUniform1i( GLES20.glGetUniformLocation(program, "v_tex"), 2);
    } else {
        GLES20.glUniform1i( GLES20.glGetUniformLocation(program, "tex"), 0);
    }
    
    texMatrixLocation = GLES20.glGetUniformLocation(program, "tex_mat");
    inPosLocation = GLES20.glGetAttribLocation(program, "in_pos");
    inTcLocation = GLES20.glGetAttribLocation(program, "in_tc")
    
    
    GLES20.glUseProgram(program);
    
    // Upload the vertex coordinates.
    GLES20.glEnableVertexAttribArray(inPosLocation);
    GLES20.glVertexAttribPointer(inPosLocation, 
                                 /* size= */ 2,
                                 /* type= */ GLES20.GL_FLOAT, 
                                 /* normalized= */ false, 
                                 /* stride= */ 0,
                                 FULL_RECTANGLE_BUFFER);

    // Upload the texture coordinates.
    GLES20.glEnableVertexAttribArray(inTcLocation);
    GLES20.glVertexAttribPointer(inTcLocation, 
                                 /* size= */ 2,
                                 /* type= */ GLES20.GL_FLOAT, 
                                 /* normalized= */ false, 
                                 /* stride= */ 0,
                                 FULL_RECTANGLE_TEXTURE_BUFFER);

    // Upload the texture transformation matrix.
    GLES20.glUniformMatrix4fv( texMatrixLocation, 
                               1 /* count= */, 
                               false /* transpose= */, 
                               texMatrix, 
                               0 /* offset= */);
    
    
    
    // Bind the texture.
    if( shaderType == ShaderType.OES ) {
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId[0]);
    } else if( shaderType == ShaderType.RGB ) {
        GLES20.glActiveTexture(GLES20.GL_TEXTRUE0);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId[0]);
    } else if( shaderType == ShaderType.YUV ) {
    	GLES20.glActiveTexture(GLES20.GL_TEXTRUE0);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId[0]);
        GLES20.glActiveTexture(GLES20.GL_TEXTRUE1);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId[1]);
        GLES20.glActiveTexture(GLES20.GL_TEXTRUE2);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId[2]);
    }
    
    
    
    // Draw the texture.
    GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight);
    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
    
    
    // Unbind the texture as a precaution.
    if( shaderType == ShaderType.OES ) {
        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
    } else if( shaderType == ShaderType.RGB ) {
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
    } else if( shaderType == ShaderType.YUV ) {
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
        GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
        GLES20.glActiveTexture(GLES20.GL_TEXTURE2);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
    }

 

 

 

 

 

텍스처 포맷별 I420 변환을 위한 fragment shader source

RGB

precision medium float;

varying vec2 tc;
uniform sampler2D tex;
uniform vec2 xUnit;
uniform vec4 coeffs;

void mina() {
    gl_FragColor.r = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc - 1.5 * xUnit).rgb);
    gl_FragColor.g = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc - 0.5 * xUnit).rgb);
    gl_FragColor.b = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc + 0.5 * xUnit).rgb);
    gl_FragColor.a = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc + 1.5 * xUnit).rgb);
}

 

YUV

precision medium float;

varying vec2 tc;
uniform sampler2D y_tex;
uniform sampler2D u_tex;
uniform sampler2D v_tex;

vec4 sample(vec2 p) {
    float y = texture2D(y_tex, p).r * 1.16438;
    float u = texture2D(u_tex, p).r;
    float v = texture2D(v_tex, p).r;
    return vec4(y + 1.59603 * v - 0.874202, 
                y - 0.391762 * u - 0.812968 * v + 0.531668,
                y + 2.01723 * u - 1.08563, 1);
}

uniform vec2 xUnit;
uniform vec4 coeffs;

void mina() {
    gl_FragColor.r = coeffs.a + dot(coeffs.rgb, sample(tc - 1.5 * xUnit).rgb);
    gl_FragColor.g = coeffs.a + dot(coeffs.rgb, sample(tc - 0.5 * xUnit).rgb);
    gl_FragColor.b = coeffs.a + dot(coeffs.rgb, sample(tc + 0.5 * xUnit).rgb);
    gl_FragColor.a = coeffs.a + dot(coeffs.rgb, sample(tc + 1.5 * xUnit).rgb);
}

 

OES Texture

#extension GL_OES_EGL_image_external : require
precision medium float;

varying vec2 tc;
uniform samplerExternnalOES tex;
uniform vec2 xUnit;
uniform vec4 coeffs;

void mina() {
    gl_FragColor.r = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc - 1.5 * xUnit).rgb);
    gl_FragColor.g = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc - 0.5 * xUnit).rgb);
    gl_FragColor.b = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc + 0.5 * xUnit).rgb);
    gl_FragColor.a = coeffs.a + dot(coeffs.rgb, texture2D(tex, tc + 1.5 * xUnit).rgb);
}

 

    // Y'UV444 to RGB888, see https://en.wikipedia.org/wiki/YUV#Y%E2%80%B2UV444_to_RGB888_conversion
    // We use the ITU-R BT.601 coefficients for Y, U and V.
    // The values in Wikipedia are inaccurate, the accurate values derived from the spec are:
    // Y = 0.299 * R + 0.587 * G + 0.114 * B
    // U = -0.168736 * R - 0.331264 * G + 0.5 * B + 0.5
    // V = 0.5 * R - 0.418688 * G - 0.0813124 * B + 0.5
    // To map the Y-values to range [16-235] and U- and V-values to range [16-240], the matrix has
    // been multiplied with matrix:
    // {{219 / 255, 0, 0, 16 / 255},
    // {0, 224 / 255, 0, 16 / 255},
    // {0, 0, 224 / 255, 16 / 255},
    // {0, 0, 0, 1}}
    private static final float[] yCoeffs =
        new float[] {0.256788f, 0.504129f, 0.0979059f, 0.0627451f};
    private static final float[] uCoeffs =
        new float[] {-0.148223f, -0.290993f, 0.439216f, 0.501961f};
    private static final float[] vCoeffs =
        new float[] {0.439216f, -0.367788f, -0.0714274f, 0.501961f};