PYMOL-3013: Fix unit cell rendering

schrodinger · Jan 30, 2024 · 05c920b · 05c920b
1 parent 028cd2f
commit 05c920b
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 6 deletions.
diff --git a/layer2/CoordSet.cpp b/layer2/CoordSet.cpp
@@ -1285,6 +1285,16 @@ void CoordSet::update(int state)
   if ((Obj->visRep & cRepCellBit) && !UnitCellCGO) {
     if (auto const* sym = getSymmetry()) {
       UnitCellCGO.reset(CrystalGetUnitCellCGO(&sym->Crystal));
+      auto use_shader = SettingGet<bool>(G, cSetting_use_shaders);
+      if (use_shader) {
+        auto color = ColorGet(G, Obj->Color);
+        auto preCGO = pymol::make_unique<CGO>(G);
+        CGOColorv(preCGO.get(), color);
+        CGOAppendNoStop(preCGO.get(), UnitCellCGO.get());
+        std::unique_ptr<CGO> optimized(CGOOptimizeToVBONotIndexed(preCGO.get(), 0));
+        UnitCellShaderCGO.reset(optimized.release());
+        assert(UnitCellShaderCGO->use_shader);
+      }
     }
   }
 
@@ -1322,6 +1332,7 @@ void CoordSet::render(RenderInfo * info)
   const auto pass = info->pass;
   const auto pick = bool(info->pick);
   auto* ray = info->ray;
+  auto use_shader = SettingGet<bool>(G, cSetting_use_shaders);
 
   if (!(ray || pick) &&
       (SettingGet<int>(*this, cSetting_defer_builds_mode) == 5)) {
@@ -1371,7 +1382,11 @@ void CoordSet::render(RenderInfo * info)
     } else if (!pick && pass == RenderPass::Opaque && G->HaveGUI &&
                G->ValidContext) {
       ObjectUseColor(Obj);
-      CGORender(UnitCellCGO.get(), ColorGet(G, Obj->Color), Setting.get(),
+      auto renderCGO = UnitCellCGO.get();
+      if (use_shader && UnitCellShaderCGO) {
+        renderCGO = UnitCellShaderCGO.get();
+      }
+      CGORender(renderCGO, ColorGet(G, Obj->Color), Setting.get(),
           Obj->Setting.get(), info, nullptr);
     }
   }

diff --git a/layer2/CoordSet.h b/layer2/CoordSet.h
@@ -134,6 +134,7 @@ struct CoordSet : CObjectState {
   CGO *SculptCGO = nullptr;
   CGO *SculptShaderCGO = nullptr;
   pymol::cache_ptr<CGO> UnitCellCGO;
+  pymol::cache_ptr<CGO> UnitCellShaderCGO;
 
   MapType *Coord2Idx = nullptr;
   float Coord2IdxReq = 0, Coord2IdxDiv = 0;

diff --git a/layer2/ObjectSurface.cpp b/layer2/ObjectSurface.cpp
@@ -866,18 +866,25 @@ static void ObjectSurfaceRenderRay(PyMOLGlobals * G, ObjectSurface *I,
 static void ObjectSurfaceRenderCell(PyMOLGlobals *G, ObjectSurface * I,
     RenderInfo * info, ObjectSurfaceState *ms, short use_shader)
 {
+  /**
+   * TODO: Ray with primitive CGO
+   */
+
   const float *color = ColorGet(G, I->Color);
   if (use_shader != ms->UnitCellCGO->has_draw_buffers){
     if (use_shader){
-      CGO *convertcgo = CGOOptimizeToVBONotIndexed(ms->UnitCellCGO.get(), 0);
-      ms->UnitCellCGO.reset(convertcgo);
-      assert(ms->UnitCellCGO->use_shader);
+      auto preCGO = pymol::make_unique<CGO>(G);
+      CGOColorv(preCGO.get(), color);
+      CGOAppendNoStop(preCGO.get(), ms->UnitCellCGO.get());
+      std::unique_ptr<CGO> optimized(CGOOptimizeToVBONotIndexed(preCGO.get(), 0));
+      ms->UnitCellShaderCGO.reset(optimized.release());
+      assert(ms->UnitCellShaderCGO->use_shader);
     } else {
       ms->UnitCellCGO.reset(CrystalGetUnitCellCGO(&ms->Crystal));
     }
   }
-  CGORender(ms->UnitCellCGO.get(), color,
-              I->Setting.get(), NULL, info, NULL);
+  auto renderCGO = use_shader ? ms->UnitCellShaderCGO.get() : ms->UnitCellCGO.get();
+  CGORender(renderCGO, color, I->Setting.get(), nullptr, info, nullptr);
 }
 
 void ObjectSurface::render(RenderInfo * info)

diff --git a/layer2/ObjectSurface.h b/layer2/ObjectSurface.h
@@ -51,6 +51,7 @@ struct ObjectSurfaceState : public CObjectState
   cIsosurfaceMode Mode;
   int DotFlag;
   pymol::cache_ptr<CGO> UnitCellCGO;
+  pymol::cache_ptr<CGO> UnitCellShaderCGO;
   cIsosurfaceSide Side = cIsosurfaceSide::front;
   pymol::cache_ptr<CGO> shaderCGO;
   ObjectSurfaceState(PyMOLGlobals* G);