rook-ceph: Allow user to provide sub-component resources

This commit adds a new parameter `resources` for sub-components' cpu and
memory requests and limits.

Signed-off-by: Suraj Deshmukh <suraj@kinvolk.io>

docs/configuration-reference/components/rook-ceph.md

@@ -50,8 +50,28 @@ component "rook-ceph" {
     default        = true
     reclaim_policy = "Delete"
   }
-}
 
+  resources {
+    osd {
+      requests {
+        cpu    = "5"
+        memory = "5Gi"
+      }
+
+      limits {
+        cpu    = "5"
+        memory = "5Gi"
+      }
+    }
+
+    mon {}
+    mgr {}
+    mds {}
+    prepareosd {}
+    crashcollector {}
+    mgr_sidecar {}
+  }
+}
 ```
 
 The Ceph cluster needs to be deployed in the same namespace as the Rook operator at the moment.
@@ -73,6 +93,13 @@ Table of all the arguments accepted by the component.
 | `storage_class.enable`         | Install Storage Class config.                                                                                                                                                                                                                                                                                               |   false  | bool                                                                                                           |  false   |
 | `storage_class.default`        | Make this Storage Class as a default one.                                                                                                                                                                                                                                                                                   |   false  | bool                                                                                                           |  false   |
 | `storage_class.reclaim_policy` | Persistent volumes created with this storage class will have this reclaim policy. This field decides what happens to the volume after a user deletes a PVC. Valid values: `Retain`, `Recycle` and `Delete`. Read more in the [Kubernetes docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming). | `Retain` | string                                                                                                         |  false   |
+| `resources.osd`                | Resource request and/or limits for OSDs.                                                                                                                                                                                                                                                                                    |     -    | object(request {cpu = string, memory = string}, limits {cpu = string, memory = string})                        |  false   |
+| `resources.mon`                | Resource request and/or limits for MONs.                                                                                                                                                                                                                                                                                    |     -    | object(request {cpu = string, memory = string}, limits {cpu = string, memory = string})                        |  false   |
+| `resources.mgr`                | Resource request and/or limits for MGRs.                                                                                                                                                                                                                                                                                    |     -    | object(request {cpu = string, memory = string}, limits {cpu = string, memory = string})                        |  false   |
+| `resources.mds`                | Resource request and/or limits for MDS.                                                                                                                                                                                                                                                                                     |     -    | object(request {cpu = string, memory = string}, limits {cpu = string, memory = string})                        |  false   |
+| `resources.prepareosd`         | Resource request and/or limits for OSD prepare job.                                                                                                                                                                                                                                                                         |     -    | object(request {cpu = string, memory = string}, limits {cpu = string, memory = string})                        |  false   |
+| `resources.crashcollector`     | Resource request and/or limits for Crashcollector.                                                                                                                                                                                                                                                                          |     -    | object(request {cpu = string, memory = string}, limits {cpu = string, memory = string})                        |  false   |
+| `resources.mgr_sidecar`        | Resource request and/or limits for MGR sidecar.                                                                                                                                                                                                                                                                             |     -    | object(request {cpu = string, memory = string}, limits {cpu = string, memory = string})                        |  false   |
 
 ## Applying
 

pkg/components/rook-ceph/component.go

@@ -41,6 +41,27 @@ type component struct {
 	TolerationsRaw string
 	StorageClass   *StorageClass `hcl:"storage_class,block"`
 	EnableToolbox  bool          `hcl:"enable_toolbox,optional"`
+
+	Resources *Resources `hcl:"resources,block"`
+}
+
+// Resources struct allows user to specify resource request and limits on the rook-ceph
+// sub-components.
+type Resources struct {
+	MON               *util.ResourceRequirements `hcl:"mon,block"`
+	MONRaw            string
+	MGR               *util.ResourceRequirements `hcl:"mgr,block"`
+	MGRRaw            string
+	OSD               *util.ResourceRequirements `hcl:"osd,block"`
+	OSDRaw            string
+	MDS               *util.ResourceRequirements `hcl:"mds,block"`
+	MDSRaw            string
+	PrepareOSD        *util.ResourceRequirements `hcl:"prepareosd,block"`
+	PrepareOSDRaw     string
+	CrashCollector    *util.ResourceRequirements `hcl:"crashcollector,block"`
+	CrashCollectorRaw string
+	MGRSidecar        *util.ResourceRequirements `hcl:"mgr_sidecar,block"`
+	MGRSidecarRaw     string
 }
 
 // StorageClass provides struct to enable it or make it default.
@@ -69,6 +90,44 @@ func (c *component) LoadConfig(configBody *hcl.Body, evalContext *hcl.EvalContex
 	return gohcl.DecodeBody(*configBody, evalContext, c)
 }
 
+func (c *component) addResourceRequirements() error {
+	if c.Resources == nil {
+		return nil
+	}
+
+	var err error
+
+	if c.Resources.MONRaw, err = util.RenderResourceRequirements(c.Resources.MON); err != nil {
+		return fmt.Errorf("rendering resources.mon: %w", err)
+	}
+
+	if c.Resources.MGRRaw, err = util.RenderResourceRequirements(c.Resources.MGR); err != nil {
+		return fmt.Errorf("rendering resources.mgr: %w", err)
+	}
+
+	if c.Resources.OSDRaw, err = util.RenderResourceRequirements(c.Resources.OSD); err != nil {
+		return fmt.Errorf("rendering resources.osd: %w", err)
+	}
+
+	if c.Resources.MDSRaw, err = util.RenderResourceRequirements(c.Resources.MDS); err != nil {
+		return fmt.Errorf("rendering resources.mds: %w", err)
+	}
+
+	if c.Resources.PrepareOSDRaw, err = util.RenderResourceRequirements(c.Resources.PrepareOSD); err != nil {
+		return fmt.Errorf("rendering resources.prepareosd: %w", err)
+	}
+
+	if c.Resources.CrashCollectorRaw, err = util.RenderResourceRequirements(c.Resources.CrashCollector); err != nil {
+		return fmt.Errorf("rendering resources.crashcollector: %w", err)
+	}
+
+	if c.Resources.MGRSidecarRaw, err = util.RenderResourceRequirements(c.Resources.MGRSidecar); err != nil {
+		return fmt.Errorf("rendering resources.mgr_sidecar: %w", err)
+	}
+
+	return nil
+}
+
 // TODO: Convert to Helm chart.
 func (c *component) RenderManifests() (map[string]string, error) {
 	// Generate YAML for Ceph cluster.
@@ -78,6 +137,10 @@ func (c *component) RenderManifests() (map[string]string, error) {
 		return nil, fmt.Errorf("rendering tolerations: %w", err)
 	}
 
+	if err := c.addResourceRequirements(); err != nil {
+		return nil, fmt.Errorf("rendering resources field: %w", err)
+	}
+
 	ret := make(map[string]string)
 
 	// Parse template with values

pkg/components/rook-ceph/component_test.go

@@ -123,6 +123,139 @@ func TestConversion(t *testing.T) {
 			expected: "Delete",
 			fn:       testutil.MatchJSONPathStringValue,
 		},
+		{
+			name: "resources_osd_requests_cpu",
+			inputConfig: `component "rook-ceph" {
+				resources {
+					osd {
+						requests {
+							cpu = "5"
+						}
+					}
+				}
+			}`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "ceph.rook.io/v1", Kind: "CephCluster", Name: "rook-ceph",
+			},
+			jsonPath: "{.spec.resources.osd}",
+			expected: "{\"requests\":{\"cpu\":\"5\"}}",
+			fn:       testutil.MatchJSONPathJSONValue,
+		},
+		{
+			name: "resources_mgr_requests_memory",
+			inputConfig: `component "rook-ceph" {
+				resources {
+					mgr {
+						requests {
+							memory = "10Gi"
+						}
+					}
+				}
+			}`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "ceph.rook.io/v1", Kind: "CephCluster", Name: "rook-ceph",
+			},
+			jsonPath: "{.spec.resources.mgr}",
+			expected: "{\"requests\":{\"memory\":\"10Gi\"}}",
+			fn:       testutil.MatchJSONPathJSONValue,
+		},
+		{
+			name: "resources_mon_limits_cpu",
+			inputConfig: `component "rook-ceph" {
+				resources {
+					mon {
+						limits {
+							cpu = "3"
+						}
+					}
+				}
+			}`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "ceph.rook.io/v1", Kind: "CephCluster", Name: "rook-ceph",
+			},
+			jsonPath: "{.spec.resources.mon}",
+			expected: "{\"limits\":{\"cpu\":\"3\"}}",
+			fn:       testutil.MatchJSONPathJSONValue,
+		},
+		{
+			name: "resources_mds_limits_memory",
+			inputConfig: `component "rook-ceph" {
+				resources {
+					mds {
+						limits {
+							memory = "2Gi"
+						}
+					}
+				}
+			}`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "ceph.rook.io/v1", Kind: "CephCluster", Name: "rook-ceph",
+			},
+			jsonPath: "{.spec.resources.mds}",
+			expected: "{\"limits\":{\"memory\":\"2Gi\"}}",
+			fn:       testutil.MatchJSONPathJSONValue,
+		},
+		{
+			name: "resources_prepareosd_requests",
+			inputConfig: `component "rook-ceph" {
+				resources {
+					prepareosd {
+						requests {
+							memory = "9Gi"
+							cpu    = "9"
+						}
+					}
+				}
+			}`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "ceph.rook.io/v1", Kind: "CephCluster", Name: "rook-ceph",
+			},
+			jsonPath: "{.spec.resources.prepareosd}",
+			expected: "{\"requests\":{\"cpu\":\"9\",\"memory\":\"9Gi\"}}",
+			fn:       testutil.MatchJSONPathJSONValue,
+		},
+		{
+			name: "resources_crashcollector_limits",
+			inputConfig: `component "rook-ceph" {
+				resources {
+					crashcollector {
+						limits {
+							cpu    = "7"
+							memory = "7Gi"
+						}
+					}
+				}
+			}`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "ceph.rook.io/v1", Kind: "CephCluster", Name: "rook-ceph",
+			},
+			jsonPath: "{.spec.resources.crashcollector}",
+			expected: "{\"limits\":{\"cpu\":\"7\",\"memory\":\"7Gi\"}}",
+			fn:       testutil.MatchJSONPathJSONValue,
+		},
+		{
+			name: "resources_mgr_sidecar_all",
+			inputConfig: `component "rook-ceph" {
+				resources {
+					mgr_sidecar {
+						limits {
+							cpu = "6"
+						}
+
+						requests {
+							memory = "1Gi"
+							cpu    = "1"
+						}
+					}
+				}
+			}`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "ceph.rook.io/v1", Kind: "CephCluster", Name: "rook-ceph",
+			},
+			jsonPath: "{.spec.resources.mgr-sidecar}",
+			expected: "{\"limits\":{\"cpu\":\"6\"},\"requests\":{\"cpu\":\"1\",\"memory\":\"1Gi\"}}",
+			fn:       testutil.MatchJSONPathJSONValue,
+		},
 	}
 
 	for _, tc := range testCases {

pkg/components/rook-ceph/manifests.go

@@ -75,7 +75,16 @@ spec:
       tolerations: {{ .TolerationsRaw }}
       {{- end }}
   annotations:
+  {{- if .Resources }}
   resources:
+    mon: {{ .Resources.MONRaw }}
+    mgr: {{ .Resources.MGRRaw }}
+    osd: {{ .Resources.OSDRaw }}
+    mds: {{ .Resources.MDSRaw }}
+    prepareosd: {{ .Resources.PrepareOSDRaw }}
+    crashcollector: {{ .Resources.CrashCollectorRaw }}
+    mgr-sidecar: {{ .Resources.MGRSidecarRaw }}
+  {{- end }}
   removeOSDsIfOutAndSafeToRemove: false
   storage: # cluster level storage configuration and selection
     useAllNodes: true

pkg/components/util/types.go

@@ -96,3 +96,30 @@ func (n *NodeSelector) Render() (string, error) {
 
 	return string(b), nil
 }
+
+// ResourceRequirements allows user to specify resource requests and/or limits.
+type ResourceRequirements struct {
+	Requests *ResourceList `hcl:"requests,block" json:"requests,omitempty"`
+	Limits   *ResourceList `hcl:"limits,block" json:"limits,omitempty"`
+}
+
+// ResourceList allows user to specify CPU and memory.
+type ResourceList struct {
+	CPU    string `hcl:"cpu,optional" json:"cpu,omitempty"`
+	Memory string `hcl:"memory,optional" json:"memory,omitempty"`
+}
+
+// RenderResourceRequirements takes a list of ResourceRequirements.
+// It returns a json string and an error if any.
+func RenderResourceRequirements(r *ResourceRequirements) (string, error) {
+	if r == nil {
+		return "", nil
+	}
+
+	b, err := json.Marshal(r)
+	if err != nil {
+		return "", err
+	}
+
+	return string(b), nil
+}