diff --git a/src/main/scala/scalismo/faces/render/TriangleRenderer.scala b/src/main/scala/scalismo/faces/render/TriangleRenderer.scala index d5ad4ce..4bc501c 100644 --- a/src/main/scala/scalismo/faces/render/TriangleRenderer.scala +++ b/src/main/scala/scalismo/faces/render/TriangleRenderer.scala @@ -209,16 +209,33 @@ object TriangleRenderer { /** render a correspondence image into a buffer, contains information about triangle rasterization */ def renderCorrespondence(mesh: TriangleMesh[_3D], + triangleFilter: (TriangleId) => Boolean, pointShader: PointShader, + screenTransform: InvertibleTransform3D, buffer: RenderBuffer[Option[TriangleFragment]]): RenderBuffer[Option[TriangleFragment]] = { renderMesh[Option[TriangleFragment]]( mesh, + triangleFilter, pointShader, + screenTransform, PixelShaders.CorrespondenceShader(mesh), buffer ) } + /** render a correspondence image into a buffer, contains information about triangle rasterization */ + def renderCorrespondence(mesh: TriangleMesh[_3D], + pointShader: PointShader, + buffer: RenderBuffer[Option[TriangleFragment]]): RenderBuffer[Option[TriangleFragment]] = { + renderMesh[Option[TriangleFragment]]( + mesh, + pointShader, + PixelShaders.CorrespondenceShader(mesh), + buffer + ) + } + + /** render window depth values (as used by Z buffer, in range [0, 1]) */ def renderDepthWindow(mesh: TriangleMesh[_3D], pointShader: PointShader, diff --git a/src/main/scala/scalismo/faces/sampling/face/CorrespondenceMoMoRenderer.scala b/src/main/scala/scalismo/faces/sampling/face/CorrespondenceMoMoRenderer.scala new file mode 100644 index 0000000..2034537 --- /dev/null +++ b/src/main/scala/scalismo/faces/sampling/face/CorrespondenceMoMoRenderer.scala @@ -0,0 +1,131 @@ +/* + * Copyright University of Basel, Graphics and Vision Research Group + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package scalismo.faces.sampling.face + +import scalismo.faces.color.RGBA +import scalismo.faces.image.PixelImage +import scalismo.faces.landmarks.TLMSLandmark2D +import scalismo.faces.mesh.VertexColorMesh3D +import scalismo.faces.momo.MoMo +import scalismo.faces.parameters.RenderParameter +import scalismo.faces.render.TriangleRenderer.TriangleFragment +import scalismo.faces.render.{PixelShader, TriangleFilters, TriangleRenderer, ZBuffer} +import scalismo.geometry.{Vector, _3D} +import scalismo.mesh.{MeshSurfaceProperty, SurfacePointProperty} +import scalismo.utils.Memoize + +/** Render correspondence images. Caches rasterization separately from shading. */ +class CorrespondenceMoMoRenderer(override val model: MoMo, override val clearColor: RGBA) extends MoMoRenderer(model, clearColor) { + + def renderCorrespondenceImage(parameters: RenderParameter): PixelImage[Option[TriangleFragment]] = { + val inst = instance(parameters) + val buffer = ZBuffer[Option[TriangleFragment]](parameters.imageSize.width, parameters.imageSize.height, None) + val worldMesh = inst.shape.transform(parameters.modelViewTransform.apply) + val backfaceCullingFilter = TriangleFilters.backfaceCullingFilter(worldMesh, parameters.view.eyePosition) + TriangleRenderer.renderCorrespondence(inst.shape, backfaceCullingFilter, parameters.pointShader, parameters.imageSize.screenTransform, buffer).toImage + } + + /** render the image described by the parameters */ + override def renderImage(parameters: RenderParameter): PixelImage[RGBA] = { + val correspondenceImage = renderCorrespondenceImage(parameters) + val inst = instance(parameters) + val shader: PixelShader[RGBA] = parameters.pixelShader(inst) + correspondenceImage.map{ px => if(px.isDefined) shader(px.get) else clearColor } + } + + /** get a cached version of this renderer */ + override def cached(cacheSize: Int) = new CorrespondenceMoMoRenderer(model, clearColor) { + private val imageRenderer = Memoize(super.renderImage, cacheSize) + private val correspondenceImageRenderer = Memoize(super.renderCorrespondenceImage, cacheSize) + private val meshRenderer = Memoize(super.renderMesh, cacheSize) + private val maskRenderer = Memoize((super.renderMask _).tupled, cacheSize) + private val lmRenderer = Memoize((super.renderLandmark _).tupled, cacheSize * allLandmarkIds.length) + private val instancer = Memoize(super.instance, cacheSize) + + override def renderImage(parameters: RenderParameter): PixelImage[RGBA] = imageRenderer(parameters) + override def renderCorrespondenceImage(parameters: RenderParameter): PixelImage[Option[TriangleFragment]] = correspondenceImageRenderer(parameters) + override def renderLandmark(lmId: String, parameter: RenderParameter): Option[TLMSLandmark2D] = lmRenderer((lmId, parameter)) + override def renderMesh(parameters: RenderParameter): VertexColorMesh3D = meshRenderer(parameters) + override def instance(parameters: RenderParameter): VertexColorMesh3D = instancer(parameters) + override def renderMask(parameters: RenderParameter, mask: MeshSurfaceProperty[Int]): PixelImage[Int] = maskRenderer((parameters, mask)) + } +} + +object CorrespondenceMoMoRenderer { + def apply(model: MoMo, clearColor: RGBA) = new CorrespondenceMoMoRenderer(model, clearColor) + def apply(model: MoMo) = new CorrespondenceMoMoRenderer(model, RGBA.BlackTransparent) +} + +abstract class RenderFromCorrespondenceImage[A](correspondenceMoMoRenderer: CorrespondenceMoMoRenderer) extends ParametricImageRenderer[A]{ + override def renderImage(parameters: RenderParameter): PixelImage[A] +} + +case class DepthMapRenderer(correspondenceMoMoRenderer: CorrespondenceMoMoRenderer, clearColor: RGBA = RGBA.Black) extends RenderFromCorrespondenceImage[RGBA](correspondenceMoMoRenderer: CorrespondenceMoMoRenderer) { + override def renderImage(parameters: RenderParameter): PixelImage[RGBA] = { + val correspondenceImage = correspondenceMoMoRenderer.renderCorrespondenceImage(parameters) + val depthMap = correspondenceImage.map{ px=> + if(px.isDefined){ + val frag = px.get + val tId = frag.triangleId + val bcc = frag.worldBCC + val mesh = frag.mesh + + val posModel = mesh.position(tId, bcc) + val posEyeCoordinates = parameters.modelViewTransform(posModel) + + Some((parameters.view.eyePosition-posEyeCoordinates).norm) + }else{ + None + } + } + val values = depthMap.values.toIndexedSeq.flatten + val ma = values.max + val mi = values.min + val mami = ma-mi + depthMap.map{d=> + if(d.isEmpty) + clearColor + else { + RGBA(1.0 - (d.get - mi)/mami) + } + } + } +} + +case class CorrespondenceColorImageRenderer(correspondenceMoMoRenderer: CorrespondenceMoMoRenderer, backgroundColor: RGBA = RGBA.Black) extends RenderFromCorrespondenceImage[RGBA](correspondenceMoMoRenderer){ + val reference: VertexColorMesh3D = correspondenceMoMoRenderer.model.mean + val normalizedReference: SurfacePointProperty[RGBA] = { + val extent = reference.shape.pointSet.boundingBox.extent.toBreezeVector + val min = reference.shape.pointSet.boundingBox.origin.toBreezeVector + val extV = reference.shape.pointSet.boundingBox.extent + val minV = reference.shape.pointSet.boundingBox.origin + val normalizedPoints = reference.shape.pointSet.points.map(p => (p.toBreezeVector - min) /:/ extent).map(f => Vector[_3D](f.toArray)).toIndexedSeq + SurfacePointProperty(reference.shape.triangulation, normalizedPoints.map(d=>RGBA(d.x,d.y, d.z))) + } + + override def renderImage(parameters: RenderParameter): PixelImage[RGBA] = { + val correspondenceImage = correspondenceMoMoRenderer.renderCorrespondenceImage(parameters) + correspondenceImage.map{px => + if(px.isDefined) { + val frag = px.get + normalizedReference(frag.triangleId, frag.worldBCC) + }else { + backgroundColor + } + } + } +} \ No newline at end of file diff --git a/src/test/scala/scalismo/faces/sampling/face/CorrespondenceMoMoRendererTest.scala b/src/test/scala/scalismo/faces/sampling/face/CorrespondenceMoMoRendererTest.scala new file mode 100644 index 0000000..90686c5 --- /dev/null +++ b/src/test/scala/scalismo/faces/sampling/face/CorrespondenceMoMoRendererTest.scala @@ -0,0 +1,52 @@ +/* + * Copyright University of Basel, Graphics and Vision Research Group + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package scalismo.faces.sampling.face + +import java.net.URI + +import scalismo.faces.FacesTestSuite +import scalismo.faces.color.RGBA +import scalismo.faces.parameters.{MoMoInstance, Pose, RenderParameter} +import scalismo.faces.utils.LanguageUtilities +import scalismo.geometry.Vector + +class CorrespondenceMoMoRendererTest extends FacesTestSuite { + describe("A CorrespondenceMoMoRenderer") { + lazy val randomMomo = randomGridModel(10, 5, 0.1, 5, 5, orthogonalExpressions = false) + val moMoRenderer = MoMoRenderer(randomMomo).cached(0) + val corrMoMoRenderer = CorrespondenceMoMoRenderer(randomMomo).cached(0) + val param = RenderParameter.defaultSquare. + withMoMo(MoMoInstance.zero(randomMomo, new URI("randomModel"))). + withPose(Pose(1.0, Vector(0,0,-10), 0,0,0)) + + def diffRGBA(a: RGBA, b: RGBA) = math.abs(a.r - b.r) + math.abs(a.g - b.g) + math.abs(a.b - b.b) + math.abs(a.a - b.a) + + it("renders the same as a MoMoRenderer") { + val A = moMoRenderer.renderImage(param) + val B = corrMoMoRenderer.renderImage(param) + val sum = A.zip(B).map{case(a,b)=>diffRGBA(a,b)}.values.toIndexedSeq.sum + sum should be <= 0.0 + } + + it("is approximately as fast as the MoMoRenderer") { + val t1 = LanguageUtilities.timed{for(i <- 0 until 5) moMoRenderer.renderImage(param)}._2 + val t2 = LanguageUtilities.timed{for(i <- 0 until 5) corrMoMoRenderer.renderImage(param)}._2 + t2 should be <= t1 * 1.5 + } + + } +}