[Metal] 이미지 렌더링~ 가우시안 블러

이번에는 이미지 필터링에 많이 사용되는 가우시안 블러를 적용해 보자.
블러링은 비용이 많이 드는 프로세스이기에 사용 조건에 따라 방식을 달리 적용해야 한다. 이미지 품질이 중요한 경우 원본 사이즈에 좀더 넓은 사이즈(탭)로 블러를 적용하고, 영상처리 같은 곳에선 속도를 위해 원본이미지를 축소해 블러를 적용한다.

이 샘플은 약 1/4 크기로 이미지를 축소하고, 7탭, 2 pass 작업을 총 5회 적용하였다.

 

공통타입 헤더
가우시안의 가중치 값은 초기에 미리 계산해 공통으로 사용할 데이터에 저장한다.

#include <simd/simd.h>

typedef struct {
    float tapCount;
    float gaussian[7];
} SharedData;

 

2 pass 블러 이므로 가로용, 세로용 텍스처와 파이프라인을 구성

class Renderer:NSObject {
    var device:MTLDevice!
    
    var commandQueue: MTLCommandQueue!
    
    var imageVertexBuffer: MTLBuffer!
    var sharedDataBuffer: MTLBuffer!
    var imagePipelineState: MTLRenderPipelineState!
    
    // 두개의 텍스처 및 렌더패스
    // 가로
    var renderHTargetTexture: MTLTexture?
    var renderHPassDescriptor: MTLRenderPassDescriptor!
    var renderHPipelineState: MTLRenderPipelineState!
    
    // 세로
    var renderVTargetTexture: MTLTexture?
    var renderVPassDescriptor: MTLRenderPassDescriptor!
    var renderVPipelineState: MTLRenderPipelineState!
    
    // 입력 이미지
    var imageTexture: MTLTexture?
    var imageDepthState:MTLDepthStencilState!
    
    // 화면 렌더링 쉐이더
    var imageVertexFunction: MTLFunction!
    var renderScreenFragmentFunction: MTLFunction!
    
    // 가로방향, 세로방향 쉐이더
    var renderHTextureFragmentFunction: MTLFunction!
    var renderVTextureFragmentFunction: MTLFunction!
    
    override init() {
        super.init()
        
        self.device = MTLCreateSystemDefaultDevice()
        initMetal()
    }
    
    .
    .
    .
    
 }

 

초기화

가로방향, 세로방향을 처리할 프레그먼트 쉐이더 함수와 위의 가우시안 값을 전달할 공유 데이터 버퍼 추가.

func initMetal() {
  guard let defaultLibrary = try? self.device.makeDefaultLibrary(bundle: Bundle(for: Renderer.self)) else {
    print("[Renderer.initMetal] init error")
    return
  }

  imageVertexFunction = defaultLibrary.makeFunction(name: "imageVertexFunction")
  renderScreenFragmentFunction = defaultLibrary.makeFunction(name: "swapFragmentFunction")
  renderHTextureFragmentFunction = defaultLibrary.makeFunction(name: "imageHFragmentFunction")
  renderVTextureFragmentFunction = defaultLibrary.makeFunction(name: "imageVFragmentFunction")

  self.commandQueue = self.device.makeCommandQueue()

  let size = kImagePlaneVertexData.count * MemoryLayout<Float>.size
  imageVertexBuffer = self.device.makeBuffer(bytes: kImagePlaneVertexData, length: size)
  imageVertexBuffer.label = "ImageVertexBuffer"

  // 공유데이터 버퍼
  let sharedBufferSize = (MemoryLayout<SharedData>.size & ~0xFF) + 0x100
  sharedDataBuffer = self.device.makeBuffer(length: sharedBufferSize, options: .storageModeShared)
  sharedDataBuffer.label = "SharedBuffer"



  initRederTarget()
  initSwapRender()
  initGaussianFilter()

  self.imageTexture = loadTexture(name:"sample", ext:"png")
}


func loadTexture(name:String, ext:String) -> MTLTexture? {
  let textureLoader = MTKTextureLoader(device: device)
  if let url = Bundle(for: Renderer.self).url(forResource: name, withExtension: ext) {
  let texture = try? textureLoader.newTexture(URL: url , options: nil)
    return texture
  }
  return nil
}

 

렌더타겟 설정

원본이미지 -> 리사이징, 가로 처리 -> 세로처리 -> 화면에 렌더링을 진행하게 되며, 블러처리에 사용될 가로,세로 텍스처를 생성하고, 해당 텍스처로의 렌더 패스를 설정한다.

func initRederTarget() {
  let imageVertexDescriptor = MTLVertexDescriptor()
  imageVertexDescriptor.attributes[0].format = .float2
  imageVertexDescriptor.attributes[0].offset = 0
  imageVertexDescriptor.attributes[0].bufferIndex = 0
  imageVertexDescriptor.attributes[1].format = .float2
  imageVertexDescriptor.attributes[1].offset = 8
  imageVertexDescriptor.attributes[1].bufferIndex = 0
  imageVertexDescriptor.layouts[0].stride = 16
  imageVertexDescriptor.layouts[0].stepRate = 1
  imageVertexDescriptor.layouts[0].stepFunction = .perVertex


  let imageHPipelineDescriptor = MTLRenderPipelineDescriptor()
  imageHPipelineDescriptor.label = "ImageHRenderPipeline"
  imageHPipelineDescriptor.sampleCount = 1
  imageHPipelineDescriptor.vertexFunction = imageVertexFunction
  imageHPipelineDescriptor.fragmentFunction = renderHTextureFragmentFunction
  imageHPipelineDescriptor.vertexDescriptor = imageVertexDescriptor
  imageHPipelineDescriptor.depthAttachmentPixelFormat = .invalid
  imageHPipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm

  do {
    try self.renderHPipelineState = self.device.makeRenderPipelineState(descriptor: imageHPipelineDescriptor)
  } catch let error {
    print("error=\(error.localizedDescription)")
  }

  let imageVPipelineDescriptor = MTLRenderPipelineDescriptor()
  imageVPipelineDescriptor.label = "ImageVRenderPipeline"
  imageVPipelineDescriptor.sampleCount = 1
  imageVPipelineDescriptor.vertexFunction = imageVertexFunction
  imageVPipelineDescriptor.fragmentFunction = renderVTextureFragmentFunction
  imageVPipelineDescriptor.vertexDescriptor = imageVertexDescriptor
  imageVPipelineDescriptor.depthAttachmentPixelFormat = .invalid
  imageVPipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm

  do {
  try self.renderVPipelineState = self.device.makeRenderPipelineState(descriptor: imageVPipelineDescriptor)
  } catch let error {
  print("error=\(error.localizedDescription)")
  }


  let texDescriptor = MTLTextureDescriptor()
  texDescriptor.textureType = MTLTextureType.type2D
  texDescriptor.width = 256
  texDescriptor.height = 256
  texDescriptor.pixelFormat = .bgra8Unorm
  texDescriptor.storageMode = .private
  texDescriptor.usage = [.renderTarget, .shaderRead]

  self.renderHTargetTexture = self.device.makeTexture(descriptor: texDescriptor)
  self.renderVTargetTexture = self.device.makeTexture(descriptor: texDescriptor)

  let clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1)

  self.renderHPassDescriptor = MTLRenderPassDescriptor()
  self.renderHPassDescriptor.colorAttachments[0].texture = self.renderHTargetTexture
  self.renderHPassDescriptor.colorAttachments[0].loadAction = .clear
  self.renderHPassDescriptor.colorAttachments[0].clearColor = clearColor
  self.renderHPassDescriptor.colorAttachments[0].storeAction = .store


  self.renderVPassDescriptor = MTLRenderPassDescriptor()
  self.renderVPassDescriptor.colorAttachments[0].texture = self.renderVTargetTexture
  self.renderVPassDescriptor.colorAttachments[0].loadAction = .clear
  self.renderVPassDescriptor.colorAttachments[0].clearColor = clearColor
  self.renderVPassDescriptor.colorAttachments[0].storeAction = .store

}

 

화면 렌더링 설정

func initSwapRender() {
  let imageVertexDescriptor = MTLVertexDescriptor()
  imageVertexDescriptor.attributes[0].format = .float2
  imageVertexDescriptor.attributes[0].offset = 0
  imageVertexDescriptor.attributes[0].bufferIndex = 0
  imageVertexDescriptor.attributes[1].format = .float2
  imageVertexDescriptor.attributes[1].offset = 8
  imageVertexDescriptor.attributes[1].bufferIndex = 0
  imageVertexDescriptor.layouts[0].stride = 16
  imageVertexDescriptor.layouts[0].stepRate = 1
  imageVertexDescriptor.layouts[0].stepFunction = .perVertex


  let imagePipelineDescriptor = MTLRenderPipelineDescriptor()
  imagePipelineDescriptor.label = "ImageRenderPipeline"
  imagePipelineDescriptor.sampleCount = 1
  imagePipelineDescriptor.vertexFunction = imageVertexFunction
  imagePipelineDescriptor.fragmentFunction = renderScreenFragmentFunction
  imagePipelineDescriptor.vertexDescriptor = imageVertexDescriptor
  imagePipelineDescriptor.depthAttachmentPixelFormat = .depth32Float
  imagePipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm

  do {
    try self.imagePipelineState = self.device.makeRenderPipelineState(descriptor: imagePipelineDescriptor)
  } catch let error {
    print("error=\(error.localizedDescription)")
  }

  // depth state
  let depthDescriptor = MTLDepthStencilDescriptor()
  depthDescriptor.depthCompareFunction = .lessEqual
  depthDescriptor.isDepthWriteEnabled = true
  self.imageDepthState = self.device.makeDepthStencilState(descriptor: depthDescriptor)
}

 

가우시안 가중치 계산

func initGaussianFilter() {
  // 쉐이더에 공통적으로 전달할 데이터 생성
  // 시그마에 따른 가우시안
  let SIGMA = 4.0     // sigma^2
  let PI2 = 6.28319 // 2pi
  let TAP = 7

  let data = self.sharedDataBuffer.contents().assumingMemoryBound(to: SharedData.self)
  data.pointee.tapCount = Float(TAP)

  var total:Double = 0
  var result = [Double](repeating: 0.0, count: TAP)
  for i in 0..<TAP {
    let x = Double(i - (TAP - 1) / 2)
    result[i] = (1 / sqrtl(PI2 * SIGMA))*(expl( -(x*x) / (2*SIGMA)))
    print("\(x)=\(result[i])")
    total += result[i]
  }
  print("total=\(total)")


  // 버퍼 데이터에 저장
  // 저장시 합산이 1이 되도록 정규화
  withUnsafeMutablePointer(to: &data.pointee.gaussian) { pointer in
    pointer.withMemoryRebound(to: Float.self, capacity: TAP) { buffer in
      var index = 0
      for value in result {
        buffer[index] = Float(value) / Float(total)
        index += 1
      }
    }
  }

  print("\(data.pointee.gaussian)")
}

 

렌더링

블러는 2 pass 로 이루어지며, 2 pass 만으로는 원하는 결과가 나오지 않으므로, 해당 루틴을 몇번 반복해 주어야 한다. 
샘플에서는 5회 반복해 진행.
처음엔 원본 이미지를 이용해 그리고, 이후에는 Horizontal 작업 텍스처와, Vertical 작업 텍스처를 바꿔가며 블러를 진행한다.
마지막으로 작업된 텍스처를 화면에 렌더링.

func render(view:MTKView) {
  print("render")
  let startTime = Int64((Date().timeIntervalSince1970 * 1000.0).rounded())
  guard let renderPass = view.currentRenderPassDescriptor else { return }
  guard let drawable = view.currentDrawable else { return }
  guard let commandBuffer = self.commandQueue.makeCommandBuffer() else { return }
  commandBuffer.label = "RenderCommand"


  // 블러 추가
  for i in 0..<5 {
    var inputTexture = self.renderVTargetTexture
    if i == 0 {
      inputTexture = self.imageTexture
    }

    if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: self.renderHPassDescriptor)  {
      encoder.label = "RenderHEncoder"
      encoder.setCullMode(.front)
      encoder.setRenderPipelineState(self.renderHPipelineState)
      encoder.setVertexBuffer(self.imageVertexBuffer, offset: 0, index: 0)
      encoder.setFragmentTexture(inputTexture, index: 0)
      encoder.setFragmentBuffer(self.sharedDataBuffer, offset: 0, index: 0)
      encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
      encoder.endEncoding()
    }


    if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: self.renderVPassDescriptor)  {
      encoder.label = "RenderVEncoder"
      encoder.setCullMode(.front)
      encoder.setRenderPipelineState(self.renderVPipelineState)
      encoder.setVertexBuffer(self.imageVertexBuffer, offset: 0, index: 0)
      encoder.setFragmentTexture(self.renderHTargetTexture!, index: 0)
      encoder.setFragmentBuffer(self.sharedDataBuffer, offset: 0, index: 0)
      encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
      encoder.endEncoding()
    }
  }



  if let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPass) {
    encoder.label = "SwapEncoder"
    encoder.setCullMode(.front)
    encoder.setRenderPipelineState(self.imagePipelineState)
    encoder.setDepthStencilState(self.imageDepthState)
    encoder.setVertexBuffer(self.imageVertexBuffer, offset: 0, index: 0)
    encoder.setFragmentTexture(self.renderVTargetTexture, index: 0)
    encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
    encoder.endEncoding()
  }




  commandBuffer.present(drawable)
  commandBuffer.commit()
  commandBuffer.waitUntilCompleted()

  let endTime = Int64((Date().timeIntervalSince1970 * 1000.0).rounded())
  print("complete: \(endTime - startTime)ms)")
}

 

쉐이더

블러 적용을 위한 fragment 쉐이더로 하나로 만들어 사용해도 되나 일단 샘플에선 별도로 구성
가로방향은 x축에 해당하는 주변 픽셀들을 합하고, 세로방향은 y축에 해당하는 주변 픽셀들을 합한다.
이때 합하는 값은 가우시안 가중치에 따라 계산.

fragment float4 imageHFragmentFunction(ImageOut in [[stage_in]],
                                       texture2d<float> texture1 [[texture(0)]],
                                       constant SharedData &sharedData [[buffer(0)]]) {
    
    constexpr sampler colorSampler;
    
    float step = ( 1.0f/texture1.get_width());
    
    float3 sum = float3(0.0, 0.0, 0.0);
    for (int i=0;i<sharedData.tapCount;i++) {
        int index = i - (sharedData.tapCount - 1) / 2;
        sum += texture1.sample(colorSampler, float2(in.texCoord.x - (index*step), in.texCoord.y)).rgb * sharedData.gaussian[i];
    }

    float4 color = float4(sum, 1.0);
    return color;
}

fragment float4 imageVFragmentFunction(ImageOut in [[stage_in]],
                                       texture2d<float> texture1 [[texture(0)]],
                                       constant SharedData &sharedData [[buffer(0)]]) {
    
    constexpr sampler colorSampler;
    float step = ( 1.0f/texture1.get_height());
    
    float3 sum = float3(0.0, 0.0, 0.0);
    for (int i=0;i<sharedData.tapCount;i++) {
        int index = i - (sharedData.tapCount - 1) / 2;
        sum += texture1.sample(colorSampler, float2(in.texCoord.x, in.texCoord.y - (index*step))).rgb * sharedData.gaussian[i];
    }
    
    float4 color = float4(sum, 1.0);
    return color;
}

 

기타 렌더링 쉐이더

typedef struct {
    float3 position [[attribute(0)]];
    float2 texCoord [[attribute(1)]];
} ImageVertex;

typedef struct {
    float4 position [[position]];
    float2 texCoord;
} ImageOut;


vertex ImageOut imageVertexFunction( ImageVertex in [[stage_in]]) {
    ImageOut out;
    float4 position = float4(in.position, 1.0);
    out.position = position;
    out.texCoord = in.texCoord;
    return out;
}


fragment float4 swapFragmentFunction(ImageOut in [[stage_in]], texture2d<float> texture1 [[texture(0)]]) {
    
    constexpr sampler colorSampler;
    float4 color = texture1.sample(colorSampler, in.texCoord);
    return color;
}