CVPixelBuffer, CMSampleBuffer,Data, Metal Texture, vImage

CMSampleBuffer -> CVPixelBuffer

let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)

// nano timestamp가 필요한 경우
let scale:Float64 = 1_000_000_000
let time = Int64( CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * scale)

 

CVPixelBuffer -> CMSampleBuffer

var sampleBuffer: CMSampleBuffer? = nil

let scale = CMTimeScale(1_000_000_000)
let time = CMTime(value: CMTimeValue( secondTimestamp * Double(scale)), timescale: scale)
var timimgInfo: CMSampleTimingInfo = CMSampleTimingInfo( duration: CMTime.invalid, 
                                                         presentationTimeStamp: time, 
                                                         decodeTimeStamp: CMTime.invalid)
var videoInfo: CMVideoFormatDescription? = nil

CMVideoFormatDescriptionCreateForImageBuffer(
                  allocator: nil, 
                  imageBuffer: pixelBuffer, 
                  formatDescriptionOut: &videoInfo)

CMSampleBufferCreateForImageBuffer(
                  allocator: kCFAllocatorDefault, 
                  imageBuffer: pixelBuffer, 
                  dateReady: true, 
                  makeDateReadyCallback: nil, 
                  refcon: nil, 
                  formatDescription: videoInfo!, 
                  sampleTiming: &timimgInfo, 
                  sampleBufferOut: &sampleBuffer)

 

 

CVPixelBuffer 복사

원본과 대상의 픽셀포맷과 plane이 동일한 경우

CVPixelBufferLockBaseAddress(originBuffer, .ReadOnly)
CVPixelBufferLockBaseAddress(destBuffer, CVPixelBufferLockFlags(rawValue: 0))

for plane in 0 ..< CVPixelBufferGetPlaneCount(originBuffer) {
  let dest        = CVPixelBufferGetBaseAddressOfPlane(destBuffer, plane)
  let source      = CVPixelBufferGetBaseAddressOfPlane(originBuffer, plane)
  let height      = CVPixelBufferGetHeightOfPlane(originBuffer, plane)
  let bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(originBuffer, plane)
  memcpy(dest, source, height * bytesPerRow)
}

CVPixelBufferUnlockBaseAddress(destBuffer, CVPixelBufferLockFlags(rawValue: 0))
CVPixelBufferUnlockBaseAddress(originBuffer, .ReadOnly)

 

CVPixelBuffer -> Data

CVPixelBufferLockBaseAddress( pixelBuffer, CVPixelBufferLockFlags(rawValue: 0) )

let width = CVPixelBufferGetWidth(pixelBuffer)
let format = CVPixelBufferGetPixelFormatType(pixelBuffer)

var data = Data()
for plane in 0..<CVPixelBufferGetPlaneCount(pixelBuffer) {
  let address = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, plane)
  let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, plane)
  let bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, plane)
  
  let length = bytesPerRow * height
  data.append(Data(bytes: address!, count: length))
}

CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))

 

CMSampleBuffer -> Data

let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)

CVPixelBufferLockBaseAddress(imageBuffer!, CVPixelBufferLockFlags(rawValue: 0))
let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer!)
let height = CVPixelBufferGetHeight(imageBuffer!)
let src_buff = CVPixelBufferGetBaseAddress(imageBuffer!)
let data = Data(bytes: src_buff, length: bytesPerRow * height)
CVPixelBufferUnlockBaseAddress(imageBuffer!, CVPixelBufferLockFlags(rawValue: 0))

 

 

CVPixelBuffer -> CVMetalTexture

let colorFormat: MTLPixelFormat = .r8Unorm
let width = CVPixelBufferGetWidthOfPlane( pixelBuffer, planeIndex)
let height = CVPixelBufferGetHeightOfPlane( pixelBuffer, planeIndex)
var texture: CVMetalTexture?
let status = CVMetalTextureCacheCreateTextureFromImage( 
                  nil,
                  textureCache,
                  pixelBuffer,
                  nil,
                  colorFormat,
                  width,
                  height,
                  planeIndex,
                  &texture)
                  
                  
 if status != kCVReturnSuccess {
 
 }
                  

 

위 예처럼 텍스처 캐시를 사용하는 경우 캐시는 미리 생성

var textureCache: CVMetalTextureCache!
CVMetalTextureCacheCreate( nil, nil, self.device, nil, &textureCache)

 

 

CVMetalTexture -> MTLTexture

CVMetalTextureGetTexture( cvMetalTexture)

 

 

MTLTexture 데이터를 CVPixelBuffer로 복사

CVPixelBuffer 생성

var pixelBuffer: CVPixelBuffer?

CVPixelBufferCreate( kCFAllocatorDefault,
    texture.width,
    texture.height,
    kCVPixelFormatType_32BGRA,
    nil,
    &self.pixelBuffer)

 

복사(렌더링 이후)

CVPixelBufferLockBaseAddress( pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let pixelBufferBytes = CVPixelBufferGetBaseAddress( pixelBuffer )
let bytesPerRow = CVPixelBufferGetBytesPerRow( pixelBuffer )
let region = MTLRegionMake2D(0, 0, texture.width, texture.height)

texture.getBytes( pixelBufferBytes, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0)
CVPixelBufferUnlockBaseAddress( pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))

 

CVPixelBuffer, vImage_Buffer를 사용한 크롭, 스케일링

픽셀포맷이 BGRA 인 경우만 고려

// 소스 픽셀 버퍼 크기
let srcWidth = CVPixelBufferGetWidth(srcPixelBuffer)
let srcHeight = CVPixelBufferGetHeight(srcPixelBuffer)
let srcPixelFormat = CVPixelBufferGetPixelFormatType(srcPixelBuffer)

if srcPixelFormat != kCVPixelFormatType_32BGRA {
    assertionFailure("not supported pixel format")
}

// 타겟 픽셀 버퍼 크기
let dstWidth = 256
let dstHeight = 256

let offsetX = srcWidth / 2 - dstWidth / 2
let offsetY = srcHeight / 2 - dstHeight / 2

// 타겟 픽셀 버퍼 생성
let attributes: [String: Any] = [String(kCVPixelBufferMetalCompatibilityKey): true]

var dstPixelBuffer: CVPixelBuffer?
let result = CVPixelBufferCreate(nil, 
                                 dstWidth, 
                                 dstHeight, 
                                 srcPixelFormat, 
                                 attributes as CFDictionary, 
                                 &dstPixelBuffer)
if result != kCVReturnSuccess { return }
guard let dstPixelBuffer = dstPixelBuffer else { return }


// 소스 버퍼 lock
guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(srcPixelBuffer, .readOnly) else { return }
defer { CVPixelBufferUnlockBaseAddress(srcPixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) }

// 타겟 버퍼 lock
guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(dstPixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) else { return }
defer { CVPixelBufferUnlockBaseAddress(dstPixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) }

// 
guard let srcAddress = CVPixelBufferGetBaseAddress(srcPixelBuffer) else { return }
guard let dstAddress = CVPixelBufferGetBaseAddress(dstPixelBuffer) else { return }

let srcBytesPerRow = CVPixelBufferGetBytesPerRow(srcPixelBuffer)
let dstBytesPerRow = CVPixelBufferGetBytesPerRow(dstPixelBuffer)

// vImage_Buffer 생성
var srcBuffer = vImage_Buffer(data: srcAddress,
                              height: vImagePixelCount(srcHeight),
                              width: vImagePixelCount(srcWidth),
                              rowBytes: srcBytesPerRow)
                              
var dstBuffer = vImage_Buffer(data: dstAddress,
                              height: vImagePixelCount(dstHeight),
                              width: vImagePixelCount(dstWidth),
                              rowBytes: dstBytesPerRow)
                              
// 스케일 적용
if kvImageNoError != vImageScale_ARGB8888(&srcBuffer, &dstBuffer, nil, vImage_Flags(kvImageNoFlags)) {
	return
}