},    DataSourceRef: nil,   } E0617 06:47:14.628870 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 06:47:14.628888 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 06:47:14.628896 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 06:47:14.628901 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 06:47:14.628913 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 06:47:14.629326 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 06:47:17.991935 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 06:47:18.379402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:23.402873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:28.422888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:33.457121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:38.496046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:43.516033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:48.536252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:53.587951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:47:58.608892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:03.633116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:08.656674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:13.681222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:18.704768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:23.787629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:28.818671 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:33.840280 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:38.860381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:43.880407 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:48.903678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:53.956194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:48:54.221502 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 06:48:58.975744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:04.021480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:09.155568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:14.176085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:19.196351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:24.213536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:29.238183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:34.259466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:39.314060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:44.331493 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:49.348704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:54.606454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:49:59.632234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:04.651474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:09.676404 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:14.691034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:19.712937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:24.734027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:29.937361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:31.458143 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 06:50:34.956952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:39.981631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:45.025947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:50.044559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:50:55.085148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:00.179100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:05.196676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:10.220876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:15.237583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:20.252811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:25.271765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:30.297356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:35.313741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:40.331833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:45.352707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:50.384434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:51:55.399387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:00.420991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:05.436304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:10.455169 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:15.467483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:20.484653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:25.690895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:30.710911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:35.737005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:40.755911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:45.785926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:50.803253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:52:55.915274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:00.934245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:05.957509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:10.974289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:12.998699 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 06:53:15.992895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:21.015497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:26.357779 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:31.377900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:36.398011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:41.416178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:46.441801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:51.457494 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:53:56.475608 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:01.492896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:06.511526 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:11.528246 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:16.545283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:21.561655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:26.579314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:31.596947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:36.617752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:41.637379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:46.655272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:51.673996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:54:56.695261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:01.725266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:06.740621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:11.761532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:16.786298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:21.805157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:26.821569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:31.853209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:36.874342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:41.897769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:46.927057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:51.959017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:55:56.981930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:02.013648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:07.035082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:11.291609 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 06:56:12.056484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:17.078002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:22.101186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:27.122239 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:32.142448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:37.162468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:42.179089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:47.195750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:52.309305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:56:57.329304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:02.346164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:07.366509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:12.385123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:17.405113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:22.420931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:27.687328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:32.719400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:37.739501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:42.763753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:47.781876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:52.801627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:57:57.870614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:02.889891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:07.907703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:12.932903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:17.950955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:18.237197 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 06:58:22.971162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:27.986617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:33.021872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:38.044438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:43.062332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:48.084100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:53.101942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:58:58.124201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:03.151196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:08.169890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:13.196556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:18.223996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:23.297196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:28.326502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:33.346674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:38.371466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:43.395378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:48.518560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:54.122842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 06:59:59.154351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:04.790271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:09.809823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:14.829658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:19.844482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:24.503048 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 07:00:24.868259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:25.003430 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 07:00:29.890413 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:34.908205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:39.928447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:44.945111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:49.967502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:00:54.991337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:00.032567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:05.076674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:10.099142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:15.117470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:20.144947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:25.466465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:30.486116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:35.501902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:40.519880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:45.540432 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:50.555170 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:01:55.577037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:00.595099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:05.616062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:10.634203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:14.489286 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:02:14.489526 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 07:02:14.489627 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 07:02:14.489646 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 07:02:14.489656 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 07:02:14.489661 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 07:02:14.489669 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 07:02:14.489677 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 07:02:14.489682 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 07:02:14.489724 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 07:02:14.489790 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 07:02:14.489870 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.489926 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.490023 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:02:14.490037 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 07:02:14.490046 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 07:02:14.490057 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 07:02:14.490150 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 07:02:14.490226 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:02:14.489901 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 07:02:14.490339 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 07:02:14.490358 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 07:02:14.490404 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 07:02:14.490467 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 07:02:14.490477 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 07:02:14.490483 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 07:02:14.490523 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 07:02:14.490255 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:02:14.490628 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 07:02:14.490636 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 07:02:14.490711 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 07:02:14.490724 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 07:02:14.490698 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 07:02:14.490742 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 07:02:14.490748 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.490753 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.490759 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:02:14.490767 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 07:02:14.490772 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 07:02:14.490780 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 07:02:14.490887 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 07:02:14.490893 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:02:14.490939 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:02:14.490970 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 07:02:14.491021 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 07:02:14.491032 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 07:02:14.494020 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:02:14.494153 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 07:02:14.494168 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 07:02:14.494177 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 07:02:14.494184 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 07:02:14.494191 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.494199 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.494207 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:02:14.494216 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 07:02:14.494224 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 07:02:14.494236 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 07:02:14.494247 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 07:02:14.494263 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:02:14.494269 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:02:14.494309 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 07:02:14.494319 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 07:02:14.494400 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 07:02:14.494416 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 07:02:14.600509 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 07:02:14.600549 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 07:02:14.600835 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 07:02:14.600855 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 07:02:14.600862 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 07:02:14.601014 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 07:02:14.601206 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 07:02:14.601218 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 07:02:14.601286 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.601296 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:02:14.601305 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:02:14.601439 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 07:02:14.601452 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 07:02:14.601608 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 07:02:14.601764 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 07:02:14.601774 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:02:14.601838 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:02:14.601864 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 07:02:14.601879 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 07:02:14.601885 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 07:02:14.602344 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 07:02:14.653509 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 07:02:14.653530 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 07:02:14.653552 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 07:02:14.653560 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 07:02:14.653567 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 07:02:14.653578 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 07:02:14.654037 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 07:02:15.661782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:20.685085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:22.309112 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 07:02:25.704377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:30.727197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:35.745943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:40.764831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:46.063540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:51.081762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:02:56.102312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:01.122983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:06.142759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:11.156430 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:16.175915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:21.230012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:26.250545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:28.314492 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 07:03:31.269578 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:36.286455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:41.306221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:46.327740 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:51.344259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:03:56.395036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:01.410560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:06.433718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:11.453153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:16.473460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:21.489861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:26.516053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:31.533249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:36.554918 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:41.573864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:46.598312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:51.613960 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:04:56.638817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:01.657034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:06.682211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:11.698843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:16.720163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:21.739474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:27.110431 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:32.573354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:37.596040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:42.616935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:47.637682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:52.660414 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:05:57.679463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:02.694046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:07.717164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:12.732964 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:17.751710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:22.768654 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:27.789488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:32.802897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:37.824376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:42.840875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:47.859756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:52.878313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:06:57.895795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:02.518559 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 07:07:02.916703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:07.933011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:12.951006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:17.969081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:22.993185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:28.019713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:33.039781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:37.315309 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 07:07:38.059343 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:43.079891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:48.101988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:53.129645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:07:58.155581 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:03.186317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:08.252282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:13.279289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:18.308260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:23.365566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:28.392896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:33.413860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:38.450700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:43.469224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:48.496285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:53.550403 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:08:58.570615 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:03.592751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:08.607665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:13.651449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:18.671498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:23.007287 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 07:09:23.812266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:28.922915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:33.963033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:38.983643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:44.009313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:49.030274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:54.092266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:09:59.112180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:04.141088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:09.178090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:14.201107 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:19.502456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:24.950049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:29.972563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:34.994516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:40.014362 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:45.038974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:50.062754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:10:55.077987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:00.095708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:05.113973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:10.128260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:15.145758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:20.165193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:25.186152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:30.212120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:35.228750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:40.254277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:45.271120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:50.292629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:11:55.311621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:00.370899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:05.385474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:10.410980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:15.423077 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:20.440327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:25.859908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:31.179132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:36.212945 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:41.237076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:46.258467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:48.528771 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 07:12:51.278678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:12:56.326230 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:01.346952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:06.369243 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:11.385744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:16.412184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:17.321539 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 07:13:21.426935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:26.687525 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:27.503880 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 07:13:31.707367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:36.721651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:41.741204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:46.756761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:51.772253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:13:56.787467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:01.816151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:06.831824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:11.852765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:16.867785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:21.897794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:26.912055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:31.937737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:36.961324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:41.980579 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:47.001655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:52.018273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:14:57.036440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:02.054059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:07.076232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:12.090894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:17.119280 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:22.137956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:27.154599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:32.170288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:37.193835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:42.210868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:47.232145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:52.256415 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:15:57.278977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:02.299242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:07.316201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:12.333281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:17.350201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:22.371586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:24.450865 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 07:16:27.622359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:32.642086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:37.658587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:42.678988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:47.698062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:52.713814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:16:57.730258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:02.754057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:07.771476 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:12.794052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:14.498873 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:17:14.498940 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 07:17:14.498956 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 07:17:14.498966 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 07:17:14.498981 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 07:17:14.498992 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 07:17:14.498998 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 07:17:14.499006 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 07:17:14.499014 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 07:17:14.499018 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 07:17:14.499037 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 07:17:14.499045 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 07:17:14.499052 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 07:17:14.499057 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 07:17:14.499063 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 07:17:14.499068 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 07:17:14.499073 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 07:17:14.499088 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 07:17:14.499095 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 07:17:14.499104 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.499109 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.499119 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:17:14.499128 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 07:17:14.499134 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 07:17:14.499141 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 07:17:14.499149 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 07:17:14.499154 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:17:14.499163 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:17:14.499222 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 07:17:14.499239 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 07:17:14.499244 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 07:17:14.499252 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 07:17:14.499264 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 07:17:14.499269 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 07:17:14.499274 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.499279 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.499286 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:17:14.499293 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 07:17:14.499300 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 07:17:14.499308 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 07:17:14.499317 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 07:17:14.499322 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:17:14.499327 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:17:14.499341 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 07:17:14.499357 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 07:17:14.499363 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 07:17:14.499940 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:17:14.499988 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 07:17:14.500000 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 07:17:14.500012 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 07:17:14.500018 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 07:17:14.500026 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.500034 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.500042 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:17:14.500052 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 07:17:14.500056 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 07:17:14.500066 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 07:17:14.500074 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 07:17:14.500081 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:17:14.500087 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:17:14.500123 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 07:17:14.500135 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 07:17:14.500140 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 07:17:14.500151 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 07:17:14.575364 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 07:17:14.575481 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 07:17:14.575565 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 07:17:14.575599 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 07:17:14.575640 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 07:17:14.575674 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 07:17:14.575750 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 07:17:14.575804 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 07:17:14.575835 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.575888 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:17:14.575930 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:17:14.575978 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 07:17:14.576025 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 07:17:14.576061 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 07:17:14.576112 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 07:17:14.576151 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:17:14.576205 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:17:14.576273 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 07:17:14.576314 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 07:17:14.576378 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 07:17:14.576738 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 07:17:14.635442 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 07:17:14.635465 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 07:17:14.635496 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 07:17:14.635723 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 07:17:14.635738 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 07:17:14.635755 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 07:17:14.635829 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 07:17:17.809254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:22.828093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:27.841835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:32.859880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:37.875903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:42.898351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:47.920529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:52.938247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:17:57.961439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:02.980472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:08.005436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:13.024695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:18.048636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:23.072215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:28.093686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:33.117835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:38.140293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:43.160998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:48.178435 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:53.207877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:18:58.228985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:03.252558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:08.271896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:13.293876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:18.313984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:23.344309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:28.382908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:33.409102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:38.434787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:43.466074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:48.492042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:53.543196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:19:58.560472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:03.595339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:08.615689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:13.636334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:18.651909 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:23.717520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:28.749637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:33.772142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:34.549535 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 07:20:38.887340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:43.912727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:48.932687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:53.993141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:20:59.019691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:04.039789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:09.061656 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:14.083232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:19.102932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:24.168134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:29.189991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:34.208085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:39.227532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:44.254367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:44.526800 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 07:21:49.278315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:54.330324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:21:59.549055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:04.570488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:09.603353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:14.627062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:19.643148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:24.662930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:29.682534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:34.697437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:39.715295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:44.732173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:49.758631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:54.789957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:22:59.806885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:04.829984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:09.848278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:14.339852 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 10 items received I0617 07:23:14.872681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:19.889063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:24.926111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:29.942250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:34.459434 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 07:23:34.966461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:39.984215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:45.011261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:50.028325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:23:55.053335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:00.068706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:05.099113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:10.113161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:15.134372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:20.150717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:25.172137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:30.188555 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:35.208480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:40.231481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:45.252369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:50.273236 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:24:55.325795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:00.343470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:05.362813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:10.383821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:15.403008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:20.419597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:25.492011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:30.513793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:35.532623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:40.547591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:45.563329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:50.597212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:25:55.631585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:00.656284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:05.677566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:10.696453 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:15.719749 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:20.738683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:25.793621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:30.812324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:35.836830 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:40.854073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:45.877700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:50.966268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:26:56.152681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:00.537853 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 07:27:01.170716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:06.195358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:11.220802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:16.244345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:21.266900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:26.340264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:31.353936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:36.372020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:41.386380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:46.412546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:51.431390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:27:56.453681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:01.476454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:06.494452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:11.517109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:16.537706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:21.553850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:26.576602 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:31.601722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:36.620199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:41.635525 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:46.657564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:51.698118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:28:56.715995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:01.732932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:06.748877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:11.800801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:16.819479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:21.841258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:26.902452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:31.933065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:36.947484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:41.965998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:46.984963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:51.554991 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 07:29:52.001618 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:29:57.137514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:02.158132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:07.179671 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:12.197656 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:17.218877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:22.233977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:27.266526 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:30.484521 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 07:30:32.285509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:37.304617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:42.338316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:47.362278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:52.379079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:30:57.401199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:02.415002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:07.443877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:12.465904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:17.486635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:22.504983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:27.524470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:30.362906 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 07:31:32.549491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:37.570005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:42.603729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:47.621475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:52.641465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:31:57.668447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:02.692310 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:07.711014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:12.743359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:14.499358 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:32:14.499460 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 07:32:14.499494 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 07:32:14.499508 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 07:32:14.499518 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 07:32:14.499522 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 07:32:14.499529 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 07:32:14.499536 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 07:32:14.499541 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 07:32:14.499562 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 07:32:14.499567 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 07:32:14.499575 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.499582 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.499589 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:32:14.499596 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 07:32:14.499601 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 07:32:14.499610 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 07:32:14.499620 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 07:32:14.499626 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:32:14.499640 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:32:14.499691 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 07:32:14.499708 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 07:32:14.499714 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 07:32:14.500130 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:32:14.500173 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 07:32:14.500183 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 07:32:14.500194 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 07:32:14.500203 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 07:32:14.500209 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.500214 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.500220 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:32:14.500228 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 07:32:14.500233 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 07:32:14.500242 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 07:32:14.500252 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 07:32:14.500260 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:32:14.500269 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:32:14.500306 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 07:32:14.500315 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 07:32:14.500321 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 07:32:14.500344 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 07:32:14.500446 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 07:32:14.500464 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 07:32:14.500474 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 07:32:14.500496 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 07:32:14.500501 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 07:32:14.500507 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 07:32:14.500512 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 07:32:14.500517 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 07:32:14.500532 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 07:32:14.500539 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 07:32:14.500544 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.500548 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.500555 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:32:14.500563 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 07:32:14.500569 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 07:32:14.500576 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 07:32:14.500586 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 07:32:14.500590 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:32:14.500596 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:32:14.500635 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 07:32:14.500647 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 07:32:14.500653 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 07:32:14.500660 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 07:32:14.569692 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 07:32:14.569828 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 07:32:14.569950 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 07:32:14.570431 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 07:32:14.570595 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 07:32:14.570688 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 07:32:14.570186 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 07:32:14.570759 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 07:32:14.570800 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 07:32:14.570808 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.570814 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:32:14.570860 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:32:14.570873 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 07:32:14.570879 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 07:32:14.570933 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 07:32:14.570951 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 07:32:14.570987 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:32:14.571002 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:32:14.571063 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 07:32:14.571079 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 07:32:14.571153 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 07:32:14.611834 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 07:32:14.611859 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 07:32:14.611886 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 07:32:14.612118 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 07:32:14.612131 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 07:32:14.612143 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 07:32:14.612178 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 07:32:17.764159 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:22.779778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:27.801792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:32.822032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:37.834665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:42.853353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:47.879990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:52.908166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:32:57.943200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:02.960274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:07.981054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:12.999293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:18.023131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:23.040630 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:28.059060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:33.110993 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:38.129250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:38.543300 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 07:33:43.155495 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:48.190218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:53.222394 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:33:58.243295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:03.265591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:08.285776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:13.303958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:18.325164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:23.356519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:28.381805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:33.400877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:38.429487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:43.453062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:48.469481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:53.543070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:34:58.567151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:03.595682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:08.614474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:13.640105 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:18.682661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:24.245919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:29.266349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:34.295327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:39.323027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:44.348160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:49.368355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:54.399273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:35:59.419112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:04.440372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:09.455935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:11.492325 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 07:36:14.472094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:19.493532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:19.564139 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 07:36:25.031438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:30.048714 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:35.069303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:40.085800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:45.103361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:50.120190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:36:55.151357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:00.167214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:03.369435 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 07:37:05.193324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:10.220195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:15.237532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:20.253366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:25.360167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:30.376052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:35.401580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:40.418282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:45.437378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:50.472945 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:37:55.489692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:00.540281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:05.563723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:10.586902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:15.601196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:20.620043 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:25.634929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:30.654065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:35.667273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:40.687719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:45.702104 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:50.720086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:38:55.897702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:00.914309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:05.938194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:10.958855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:15.989675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:21.003280 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:26.022940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:31.038454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:36.057603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:41.073998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:46.092086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:51.111232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:39:56.136894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:01.151783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:06.168443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:11.184676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:16.212101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:21.227775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:22.560748 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 07:40:26.257982 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:31.275094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:36.295662 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:41.320637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:46.338455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:51.364101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:40:56.380345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:01.404717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:06.426503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:11.452011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:16.469363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:21.486900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:26.801713 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 07:41:26.873927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:31.914558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:36.934388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:41.982506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:47.002176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:52.020142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:41:57.041090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:02.057841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:07.079041 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:12.098915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:17.117588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:22.135122 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:27.151026 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:32.186627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:37.209698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:42.227356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:47.251083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:52.268625 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:42:57.291402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:02.311003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:07.331498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:12.348922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:17.366904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:22.385127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:27.591156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:32.645361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:37.659957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:42.679447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:47.698736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:52.717132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:43:57.732379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:02.753574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:07.770679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:12.796977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:17.811814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:22.855292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:27.871160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:32.891697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:37.907547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:42.930209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:47.950238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:52.976125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:44:57.992110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:03.011841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:08.027912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:13.047279 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:18.064765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:23.084048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:28.102024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:32.580279 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 07:45:33.159606 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:36.506008 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 07:45:38.185855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:43.221970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:48.253362 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:53.334971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:45:58.357387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:03.392177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:08.421524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:13.446823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:18.472459 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:23.527112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:28.553244 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:33.573274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:38.600205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:39.376229 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 07:46:43.623848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:48.650687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:53.710356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:46:58.732306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:03.747547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:08.768145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:13.784586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:14.500410 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:47:14.500410 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 07:47:14.500505 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 07:47:14.500532 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 07:47:14.500542 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 07:47:14.500552 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 07:47:14.500558 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 07:47:14.500562 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 07:47:14.500568 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 07:47:14.500574 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 07:47:14.500577 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 07:47:14.500612 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 07:47:14.500618 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 07:47:14.500625 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 07:47:14.500628 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 07:47:14.500631 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 07:47:14.500634 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 07:47:14.500637 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 07:47:14.500645 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 07:47:14.500649 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 07:47:14.500656 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 07:47:14.500660 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 07:47:14.500665 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 07:47:14.500668 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:47:14.500687 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:47:14.500696 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 07:47:14.500702 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 07:47:14.500707 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 07:47:14.500715 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 07:47:14.500722 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:47:14.500729 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:47:14.500789 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 07:47:14.500793 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 07:47:14.500797 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 07:47:14.500803 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 07:47:14.500810 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 07:47:14.500813 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 07:47:14.500817 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 07:47:14.500820 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 07:47:14.500823 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 07:47:14.500828 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 07:47:14.500844 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 07:47:14.500851 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 07:47:14.500858 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 07:47:14.500862 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 07:47:14.500866 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 07:47:14.500873 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 07:47:14.500882 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 07:47:14.500886 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 07:47:14.553493 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 07:47:14.553749 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 07:47:14.553946 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 07:47:14.554471 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 07:47:14.554602 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 07:47:14.554097 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 07:47:14.554748 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 07:47:18.933333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:23.857746 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 07:47:23.980717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:29.001693 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:34.020237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:39.041807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:44.058512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:49.075530 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:54.176775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:47:59.217825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:04.242144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:09.262583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:14.301923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:19.323105 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:24.370694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:29.386952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:34.411117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:39.455301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:44.491040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:49.511252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:54.545884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:48:59.572630 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:04.594825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:09.616769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:14.640843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:19.656228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:24.684417 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:29.698443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:34.723379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:39.740272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:44.761888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:49.780848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:54.807557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:49:59.820399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:04.839635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:09.859471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:14.878535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:19.897564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:25.007485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:30.031249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:35.047808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:40.070143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:45.100279 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:50.117370 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:50:55.155330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:00.172305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:05.197126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:10.212721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:15.229132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:20.241788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:25.289707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:30.310015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:35.326732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:40.340134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:45.356446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:50.375760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:51:55.394029 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:00.412146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:05.433870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:10.452733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:15.471705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:20.492554 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:25.392156 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 07:52:25.512763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:30.528801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:35.546200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:40.563905 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:45.587270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:50.605800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:52:55.621565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:00.641424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:05.655233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:10.671850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:15.687781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:20.702676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:25.716165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:30.732156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:35.751871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:40.767184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:45.788599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:50.804627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:53:52.592241 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 07:53:55.829883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:00.845618 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:05.870353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:10.893811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:15.917610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:20.931892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:25.979350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:30.994100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:35.510102 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 07:54:36.017179 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:41.032797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:46.056264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:51.071568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:54:56.093667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:01.113853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:06.132633 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:11.147252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:16.165341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:21.180141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:26.199808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:31.219900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:36.241780 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:41.262672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:46.281520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:51.302056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:55:56.396213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:01.419956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:06.449905 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:11.470064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:16.488039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:17.877538 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 07:56:21.509008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:26.533338 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:31.558418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:36.584066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:41.606460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:46.630185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:51.650592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:56:56.853755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:01.882171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:06.900954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:11.946639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:16.961981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:21.991643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:27.207195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:32.224028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:37.246457 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:42.268074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:47.293713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:52.323062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:57:57.422711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:02.448501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:07.473319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:12.489906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:17.512421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:22.529156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:27.547987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:32.563577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:37.587332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:42.606779 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:47.630607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:52.642077 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:58:57.665472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:02.678178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:07.703228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:12.718754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:17.742295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:22.754602 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:27.407483 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 07:59:27.776318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:32.792953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:37.812738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:42.831187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:47.847637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:52.869378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 07:59:57.890008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:02.915731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:07.938791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:12.962142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:17.981283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:23.001827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:28.023092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:33.053416 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:38.070802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:43.091879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:48.110400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:53.195130 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:00:58.213817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:03.237113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:08.254941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:13.312577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:18.328293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:21.600465 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 08:01:23.352075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:28.379975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:33.397658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:38.429835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:43.449577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:48.471112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:53.492824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:01:58.513187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:03.530480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:08.549076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:13.569947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:14.501486 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:02:14.501563 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 08:02:14.501579 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 08:02:14.501599 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:02:14.501613 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:02:14.501623 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.501634 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.501642 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:02:14.501651 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:02:14.501659 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:02:14.501673 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:02:14.501694 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:02:14.501704 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:02:14.501715 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:02:14.501769 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:02:14.501780 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:02:14.501789 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:02:14.501799 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 08:02:14.501830 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:02:14.501843 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:02:14.501848 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.501852 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.501858 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:02:14.501866 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:02:14.501872 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:02:14.501881 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:02:14.501891 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:02:14.501895 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:02:14.501900 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:02:14.501919 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:02:14.501934 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:02:14.501943 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 08:02:14.501496 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:02:14.502358 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 08:02:14.502373 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 08:02:14.502383 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 08:02:14.502398 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 08:02:14.502409 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 08:02:14.502421 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 08:02:14.502428 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 08:02:14.502435 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 08:02:14.502440 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 08:02:14.502451 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 08:02:14.502460 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 08:02:14.502468 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 08:02:14.502473 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 08:02:14.502478 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 08:02:14.502484 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 08:02:14.502489 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 08:02:14.502501 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:02:14.502510 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:02:14.502517 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.502522 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.502529 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:02:14.502538 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:02:14.502548 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:02:14.502557 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:02:14.502565 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:02:14.502569 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:02:14.502576 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:02:14.502612 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:02:14.502620 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:02:14.502633 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:02:14.502641 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 08:02:14.612709 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:02:14.612857 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:02:14.612964 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:02:14.613780 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:02:14.614059 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:02:14.614165 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:02:14.613116 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 08:02:14.614362 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:02:14.614519 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:02:14.614595 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.614671 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:02:14.614753 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:02:14.614825 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:02:14.614897 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:02:14.614983 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:02:14.615066 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:02:14.615139 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:02:14.615211 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:02:14.615302 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:02:14.615381 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:02:14.615546 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 08:02:14.657840 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:02:14.658174 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:02:14.658300 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:02:14.658365 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:02:14.658409 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:02:14.658470 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:02:14.658909 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 08:02:18.598907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:23.691860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:28.716149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:33.745485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:38.769027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:43.786752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:48.858067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:54.037687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:02:59.061276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:04.086576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:07.892647 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 08:03:09.112824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:14.133888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:19.159817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:24.182260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:29.208096 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:34.229909 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:39.245647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:42.524149 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 08:03:44.264296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:49.286199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:54.375427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:03:59.393300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:04.415742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:09.446547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:14.461758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:19.484081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:24.508084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:29.527216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:34.544218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:39.576624 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:44.593154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:49.613530 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:54.441578 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 08:04:54.685640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:04:59.706632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:04.723651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:09.746372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:14.767588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:19.801149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:25.176751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:30.209336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:35.222610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:40.240184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:45.253961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:50.275773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:05:55.306225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:00.327367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:05.348918 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:10.385436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:15.408012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:20.426778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:25.448500 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:30.467846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:35.484748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:40.500705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:45.579574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:50.604457 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:06:55.621501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:00.649316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:05.667514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:10.686187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:15.703218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:20.719734 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:25.865075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:30.898772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:35.917159 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:40.940998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:45.959040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:50.981708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:07:56.557718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:01.583698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:06.606488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:11.625084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:16.647932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:21.672152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:26.778632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:31.797375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:34.603519 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 08:08:36.868697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:41.904460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:46.921092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:51.950996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:08:56.966184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:01.983927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:06.997038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:12.018437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:17.035045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:22.053230 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:27.090378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:32.103816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:37.127501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:42.146344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:47.173439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:52.190470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:09:57.223167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:01.543252 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 08:10:02.243648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:07.262315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:12.402255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:17.435016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:22.454116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:27.471899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:27.903022 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 08:10:32.492625 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:37.511580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:42.529265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:47.548001 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:52.573710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:10:57.591127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:02.618900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:07.635232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:12.662141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:17.681361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:22.703631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:27.719590 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:32.741767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:37.757215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:42.776398 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:47.798523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:52.820466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:11:57.842424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:02.862254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:07.883959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:12.906955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:17.931576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:22.951322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:27.974499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:32.994610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:38.019965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:43.035086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:48.058154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:53.076464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:12:58.098630 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:03.121608 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:08.161610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:13.191016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:18.213571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:23.805344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:28.982264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:34.009944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:39.030537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:44.058850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:49.091018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:54.112756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:13:59.128816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:00.495318 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 11 items received I0617 08:14:04.146939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:09.170455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:14.192668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:19.208412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:24.233995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:29.260644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:34.282106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:39.297228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:44.326523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:49.342907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:54.736763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:14:59.760378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:04.780380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:09.795400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:14.832311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:19.846003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:25.010373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:30.030758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:35.052634 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:40.068139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:45.085853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:50.325727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:55.353597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:15:55.610143 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 08:16:00.370130 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:05.392212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:10.413357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:15.433316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:20.448766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:25.732588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:30.747812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:35.769373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:40.786784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:45.820671 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:50.835242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:16:55.880440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:00.900445 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:05.918956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:10.939117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:13.546893 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 08:17:14.501755 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:17:14.501853 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 08:17:14.501865 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 08:17:14.501872 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:17:14.501878 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:17:14.501893 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:17:14.501902 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:17:14.501909 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.501916 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.501923 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:17:14.501931 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:17:14.501886 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.501982 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.501988 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:17:14.501994 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:17:14.502000 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:17:14.502011 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:17:14.502022 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:17:14.502028 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:17:14.502035 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:17:14.502095 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:17:14.502107 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:17:14.502112 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 08:17:14.502199 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:17:14.502210 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:17:14.502218 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:17:14.502223 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:17:14.502229 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:17:14.502256 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:17:14.502281 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:17:14.502289 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:17:14.502299 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 08:17:14.502674 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:17:14.502708 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 08:17:14.502718 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 08:17:14.502733 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 08:17:14.502742 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 08:17:14.502752 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 08:17:14.502757 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 08:17:14.502763 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 08:17:14.502770 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 08:17:14.502775 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 08:17:14.502798 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 08:17:14.502813 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 08:17:14.502820 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 08:17:14.502825 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 08:17:14.502830 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 08:17:14.503135 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 08:17:14.503182 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 08:17:14.503233 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:17:14.503277 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:17:14.503359 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.503510 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.503572 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:17:14.503678 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:17:14.503691 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:17:14.503711 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:17:14.503779 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:17:14.503862 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:17:14.503945 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:17:14.504218 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:17:14.504296 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:17:14.504352 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:17:14.504424 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 08:17:14.558883 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:17:14.558978 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:17:14.559063 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:17:14.559752 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:17:14.559861 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:17:14.559110 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 08:17:14.559953 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:17:14.560045 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:17:14.560058 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:17:14.560124 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.560214 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:17:14.560242 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:17:14.560300 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:17:14.560336 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:17:14.560398 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:17:14.560494 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:17:14.560599 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:17:14.560633 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:17:14.560689 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:17:14.560730 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:17:14.560742 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 08:17:14.602734 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:17:14.602770 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:17:14.602802 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:17:14.602811 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:17:14.602818 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:17:14.603245 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 08:17:14.603294 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:17:15.968868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:20.989333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:26.009982 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:31.027992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:36.049234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:41.067528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:46.088450 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:51.104594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:17:56.120678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:01.136658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:06.151832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:11.170013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:16.184893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:21.201771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:26.225925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:31.241265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:36.264177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:41.280914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:46.303247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:51.318653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:18:56.366882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:01.378894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:06.403934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:11.420254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:16.440764 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:21.456916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:26.807873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:31.823778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:36.851155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:38.909137 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 08:19:41.865685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:46.893235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:51.911050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:19:56.928768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:01.946562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:06.965120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:11.983082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:17.002282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:22.025083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:27.062274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:32.078147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:37.096695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:42.112643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:47.137053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:52.158710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:20:57.177837 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:02.197796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:07.217203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:12.404695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:17.431858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:22.448461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:27.470515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:32.487944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:37.509213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:40.519566 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 08:21:42.526987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:47.548379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:52.568142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:21:57.587595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:02.611553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:07.628919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:12.642944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:17.666507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:22.681109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:27.699265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:32.716143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:37.736713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:42.772703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:47.791517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:52.814837 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:22:57.837532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:02.856766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:07.877372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:12.895520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:17.923607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:22.946699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:23.730201 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 08:23:27.965712 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:32.980694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:38.000564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:43.017375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:48.036016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:53.056352 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:23:58.080047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:03.096497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:08.122267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:13.156142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:18.176876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:23.275299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:28.306059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:33.334969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:38.379782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:43.396635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:48.423225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:53.441934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:24:58.463174 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:03.494429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:08.513675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:13.526504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:18.544526 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:23.562855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:28.589345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:33.609940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:34.553280 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 08:25:38.640056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:43.670786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:48.690359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:54.466824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:25:59.487298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:04.512201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:09.531865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:14.550682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:19.577504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:24.593060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:29.617781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:34.646198 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:39.663890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:40.532305 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 08:26:44.686739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:49.705285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:54.731720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:26:59.751527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:04.776103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:09.796070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:14.819086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:19.839748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:24.861075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:29.931087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:34.952283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:39.979480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:45.007582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:50.023047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:27:55.070865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:00.100181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:05.128059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:10.144876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:15.163351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:17.920177 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 08:28:20.180694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:25.200132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:30.216105 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:35.231095 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:40.248196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:45.269281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:50.295419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:28:55.313256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:00.334319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:05.366770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:10.387625 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:15.401839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:20.426065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:25.689784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:30.709577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:35.733321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:40.754004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:45.773340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:50.787448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:29:55.810424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:00.826528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:05.845338 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:06.753465 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 08:30:10.867035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:15.883943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:20.902922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:25.919078 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:30.938708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:35.961238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:40.988310 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:46.009489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:51.029373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:30:56.071555 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:01.089926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:06.106373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:11.129152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:16.145313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:21.164874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:26.181327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:31.210220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:36.227870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:41.246391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:46.266948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:51.290826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:31:56.903164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:01.927049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:06.947214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:11.983651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:14.502641 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:32:14.502925 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 08:32:14.503044 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 08:32:14.503058 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:32:14.503064 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:32:14.503073 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.503079 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.503087 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:32:14.503110 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:32:14.503116 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:32:14.503127 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:32:14.503139 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:32:14.503144 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:32:14.503153 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:32:14.503205 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:32:14.503215 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:32:14.503221 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:32:14.503230 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 08:32:14.503534 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:32:14.503639 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:32:14.503652 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.503658 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.503666 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:32:14.503674 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:32:14.503681 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:32:14.503693 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:32:14.503713 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:32:14.503719 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:32:14.503725 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:32:14.503745 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:32:14.503761 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:32:14.503766 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 08:32:14.504466 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:32:14.504629 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 08:32:14.504738 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 08:32:14.504873 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 08:32:14.505068 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 08:32:14.505187 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 08:32:14.505305 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 08:32:14.504847 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 08:32:14.505579 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 08:32:14.505691 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 08:32:14.505778 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 08:32:14.505851 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 08:32:14.505930 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 08:32:14.505939 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 08:32:14.505964 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:32:14.505970 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:32:14.505976 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.506076 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.506090 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:32:14.506097 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:32:14.506104 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:32:14.506113 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:32:14.506124 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:32:14.506130 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:32:14.506137 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:32:14.506178 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:32:14.506186 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:32:14.506192 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:32:14.506201 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 08:32:14.505463 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 08:32:14.506366 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 08:32:14.506372 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. E0617 08:32:14.560927 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:32:14.561165 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:32:14.561262 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:32:14.561474 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:32:14.561542 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:32:14.561648 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:32:14.561732 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:32:14.561804 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:32:14.561853 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.561920 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:32:14.561991 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:32:14.562041 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:32:14.562298 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:32:14.562369 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:32:14.562446 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:32:14.561363 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 08:32:14.562482 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:32:14.562618 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:32:14.562694 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:32:14.562711 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:32:14.562717 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 08:32:14.617821 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:32:14.617844 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:32:14.617863 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:32:14.617875 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:32:14.617881 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:32:14.617893 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:32:14.618241 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 08:32:17.001330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:22.049205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:27.109771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:32.125588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:37.138258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:42.156705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:47.172639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:52.196608 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:32:57.218958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:02.235694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:07.257018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:12.272871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:17.289314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:22.315293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:27.556027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:32.571428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:34.925675 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 08:33:37.600780 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:42.633283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:47.645211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:52.792680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:33:57.806110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:02.822616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:07.838418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:12.857461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:17.871723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:22.886403 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:27.563763 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 08:34:27.901232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:32.919514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:37.935354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:42.951015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:47.965206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:48.539379 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 08:34:52.983420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:34:58.009892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:03.027710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:08.041653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:13.058949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:18.074954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:23.090024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:28.103238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:33.118362 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:38.137083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:43.161081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:48.181822 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:53.305969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:58.324290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:35:59.762815 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 1 items received I0617 08:36:03.346364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:08.362858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:13.387737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:18.410789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:23.484331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:28.497913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:33.511969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:38.535909 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:43.552008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:48.569015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:53.698443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:36:58.716847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:03.974307 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:08.991806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:14.021060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:19.036806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:24.061257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:29.077388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:34.094171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:39.112591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:44.130643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:49.146605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:54.185537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:37:59.202365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:04.222674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:09.291115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:14.311181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:19.330791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:24.375580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:29.388654 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:34.406176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:39.421945 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:44.451376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:49.468441 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:54.524702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:38:59.696456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:04.714340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:09.737803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:14.758177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:19.776274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:24.811619 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:29.828367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:32.937239 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 1 items received I0617 08:39:34.846812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:39.872094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:44.966137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:49.987026 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:39:55.189735 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:00.210967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:05.232430 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:10.248956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:15.264091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:20.281147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:25.308224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:30.325991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:35.340865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:40.356320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:45.371491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:50.386161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:40:55.402691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:00.417743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:05.438604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:10.454008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:15.473118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:20.578162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:25.621757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:30.637593 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:31.583767 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 08:41:35.657819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:40.687157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:45.702086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:50.727109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:41:55.748758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:00.763871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:05.780094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:10.806763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:15.833869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:20.847155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:25.867059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:30.895481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:34.553175 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 08:42:35.914723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:40.976006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:45.998958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:51.019517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:42:56.037378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:01.056182 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:06.080217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:11.097185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:16.116211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:21.134208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:26.157620 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:31.181304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:36.200912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:41.229102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:46.246117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:51.265991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:43:56.326212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:01.371624 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:06.397894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:11.419024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:16.434820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:21.457598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:26.477298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:31.499311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:36.515515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:37.779632 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 08:44:41.531300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:46.549523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:51.565201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:44:56.593868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:01.611190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:06.634637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:11.649860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:16.671985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:21.685966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:26.708518 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:31.726233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:36.778965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:41.796967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:46.817059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:51.845157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:45:56.985891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:02.040420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:07.061601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:12.080835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:17.100617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:22.121009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:27.142858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:32.159626 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:37.185873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:42.202635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:47.221612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:52.240103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:46:57.292415 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:02.332145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:07.350472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:12.375316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:14.503749 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:47:14.503901 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 08:47:14.503930 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 08:47:14.503991 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:47:14.504001 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:47:14.504009 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.504015 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.503969 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:47:14.504067 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:47:14.504085 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.504101 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.504214 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:47:14.504265 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:47:14.504339 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:47:14.504411 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:47:14.504164 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:47:14.504505 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:47:14.504554 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:47:14.504591 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:47:14.504646 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:47:14.504677 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:47:14.504726 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:47:14.504841 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:47:14.504938 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:47:14.505025 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 08:47:14.504808 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:47:14.505125 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:47:14.505132 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:47:14.505177 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:47:14.505184 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:47:14.505190 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:47:14.505201 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 08:47:14.504823 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 08:47:14.505255 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 08:47:14.505263 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 08:47:14.505271 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 08:47:14.505279 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 08:47:14.505287 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 08:47:14.505291 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 08:47:14.505298 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 08:47:14.505303 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 08:47:14.505307 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 08:47:14.505316 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 08:47:14.505322 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 08:47:14.505328 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 08:47:14.505332 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 08:47:14.505344 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 08:47:14.505348 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 08:47:14.505353 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 08:47:14.505362 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 08:47:14.505382 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 08:47:14.505388 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.505428 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.505436 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:47:14.505444 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 08:47:14.505448 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 08:47:14.505455 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 08:47:14.505463 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 08:47:14.505467 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:47:14.505474 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:47:14.505505 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 08:47:14.505511 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 08:47:14.505517 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 08:47:14.505524 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 08:47:14.578689 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:47:14.578713 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:47:14.578733 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:47:14.578743 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:47:14.578748 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:47:14.578762 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:47:14.578786 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 08:47:14.578797 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 08:47:14.578805 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.578815 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 08:47:14.578825 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 08:47:14.578838 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 08:47:14.578847 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 08:47:14.578859 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 08:47:14.578870 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 08:47:14.578881 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 08:47:14.578889 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 08:47:14.578915 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 08:47:14.578930 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 08:47:14.578938 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 08:47:14.579565 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 08:47:14.656457 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 08:47:14.666429 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 08:47:14.666723 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 08:47:14.666819 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 08:47:14.666907 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 08:47:14.666990 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 08:47:14.667620 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 08:47:17.395057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:22.415030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:27.436802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:32.458904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:37.478955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:42.493193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:47.515288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:52.550478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:47:57.586817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:03.211574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:08.236890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:13.253082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:14.943772 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 08:48:18.333216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:23.411228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:24.558585 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 08:48:28.489878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:33.524721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:38.539215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:43.554315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:48.577849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:53.714864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:48:58.737904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:03.765492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:08.799592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:13.876255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:18.896869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:23.966143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:27.609008 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 8 items received I0617 08:49:28.985841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:34.009930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:39.026777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:44.051485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:49.073610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:54.088604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:49:59.103748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:04.123380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:09.148515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:14.167703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:19.208035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:24.222093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:29.268579 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:34.281145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:39.298432 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:44.318796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:49.338859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:54.373741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:50:59.388260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:04.402285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:09.421318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:14.439722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:19.455165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:21.790722 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 08:51:24.475622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:29.502794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:34.518037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:39.533349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:44.546342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:49.559563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:54.578867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:51:59.596553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:04.612743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:09.646177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:14.662730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:19.716399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:24.732052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:29.757300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:34.771305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:39.791334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:44.808651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:49.833787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:54.864364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:52:59.883661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:04.902656 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:09.920140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:14.941713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:19.961186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:25.011542 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:30.033551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:35.062270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:40.127980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:45.141458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:50.156824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:53:55.172148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:00.188134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:05.212349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:10.227467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:15.239602 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:20.255112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:25.267469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:30.284246 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:35.300294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:40.323064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:45.342325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:50.360021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:54:55.378992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:00.394752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:05.409291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:10.433502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:15.455891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:20.474363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:25.488901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:30.503149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:35.520914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:40.538759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:45.566978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:50.582952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:55:55.619291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:00.634713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:05.651378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:10.667898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:13.612892 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 08:56:15.684882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:20.701181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:25.715583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:30.735609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:35.750524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:40.771244 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:45.792120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:50.808387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:56:55.824515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:00.840696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:05.857854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:07.025549 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 08:57:10.877571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:15.893508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:20.915492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:25.931767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:30.958430 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:35.974811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:40.994523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:46.017660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:51.037225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:57:56.061658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:01.076808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:03.564886 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 08:58:06.087805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:11.161729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:16.178476 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:21.204544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:26.554863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:31.574455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:36.590264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:41.603999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:42.803987 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 2 items received I0617 08:58:46.619683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:51.636402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:58:56.974154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:01.994964 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:07.015613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:12.031490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:17.052866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:22.078160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:27.096082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:32.115132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:37.135533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:42.154197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:47.170615 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:52.195813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 08:59:57.217092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:02.371798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:07.391298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:12.408544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:17.423135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:22.437958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:27.452066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:32.468140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:37.485372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:42.502861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:47.548351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:52.579397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:00:57.722134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:02.746983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:07.808364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:12.835401 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:17.849217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:22.869560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:27.899246 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:32.915075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:37.935633 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:42.953844 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:47.969366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:52.987253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:01:58.006972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:03.027265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:08.042387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:13.063406 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:14.504918 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:02:14.505056 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 09:02:14.505129 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 09:02:14.505191 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:02:14.505200 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:02:14.505210 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.505219 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.505222 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:02:14.505228 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:02:14.505227 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:02:14.505236 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.505242 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.505244 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:02:14.505248 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:02:14.505251 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:02:14.505255 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:02:14.505260 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:02:14.505263 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:02:14.505268 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:02:14.505275 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:02:14.505281 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:02:14.505287 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:02:14.505312 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 09:02:14.505320 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 09:02:14.505328 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 09:02:14.505289 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:02:14.505336 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 09:02:14.505341 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:02:14.505344 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 09:02:14.505347 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:02:14.505353 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 09:02:14.505355 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:02:14.505364 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 09:02:14.505371 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 09:02:14.505372 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 09:02:14.505377 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 09:02:14.505380 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 09:02:14.505385 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 09:02:14.505359 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 09:02:14.505390 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 09:02:14.505391 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 09:02:14.505418 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 09:02:14.505420 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 09:02:14.505276 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:02:14.505432 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:02:14.505433 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:02:14.505441 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:02:14.505446 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.505450 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.505455 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:02:14.505463 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:02:14.505468 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:02:14.505474 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:02:14.505482 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:02:14.505487 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:02:14.505437 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:02:14.505531 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:02:14.505492 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:02:14.505539 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:02:14.505545 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:02:14.505545 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:02:14.505551 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:02:14.505551 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 09:02:14.505557 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 09:02:14.571900 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:02:14.572009 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:02:14.572086 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:02:14.572120 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:02:14.572167 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:02:14.572201 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:02:14.572273 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:02:14.572326 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:02:14.572358 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.572404 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:02:14.572420 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:02:14.572438 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:02:14.572446 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:02:14.572460 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:02:14.572519 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:02:14.572526 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:02:14.573245 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 09:02:14.572534 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:02:14.573320 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:02:14.573370 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:02:14.573460 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 09:02:14.618867 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:02:14.619140 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:02:14.619232 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:02:14.619305 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:02:14.619347 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:02:14.619399 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:02:14.619799 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 09:02:18.080416 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:23.113723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:28.137743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:33.154549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:38.172101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:43.189572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:48.214154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:53.283002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:02:58.361497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:03.401241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:08.416263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:11.052888 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 09:03:13.431956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:18.446907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:23.512745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:28.531660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:33.547489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:38.560775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:43.584238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:48.601035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:52.811183 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 09:03:53.667248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:03:58.682903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:03.699559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:08.574063 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 09:04:08.712806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:13.730955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:18.872576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:22.621595 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 09:04:23.900682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:28.918549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:33.937625 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:38.975932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:43.999319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:49.020082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:54.090207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:04:59.106784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:04.125486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:09.140661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:14.158695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:19.178062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:24.236348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:29.253628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:34.271602 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:39.287376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:44.303353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:49.318235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:54.383392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:05:59.408242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:04.429921 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:09.453299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:14.486125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:19.503785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:24.529721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:29.547732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:34.566676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:39.584238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:44.604629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:49.626880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:54.648786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:06:59.663127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:04.686177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:09.704957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:14.742613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:19.771804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:24.805262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:29.824864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:34.852293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:39.878589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:44.892893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:49.917733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:54.953436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:07:59.975660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:04.997276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:10.011464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:15.031480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:20.049931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:25.066327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:30.090191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:35.110464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:40.126715 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:45.143165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:50.200709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:08:55.222571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:00.246654 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:05.259087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:10.274471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:15.297171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:20.319959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:25.366595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:30.378710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:35.394849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:40.412801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:45.428877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:50.443048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:09:55.483689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:00.499800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:05.522378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:10.542748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:15.564245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:20.578041 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:21.073687 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 09:10:25.619217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:30.635694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:35.657382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:40.671807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:45.690768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:50.703229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:10:55.970274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:00.986477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:06.007517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:11.024660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:15.638472 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 8 items received I0617 09:11:16.046019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:21.064905 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:26.084050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:31.103049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:36.123077 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:41.147673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:46.168126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:51.217383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:11:56.241233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:01.260288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:06.279928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:11.305998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:16.323566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:21.335893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:26.364282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:31.381662 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:36.407725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:41.425213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:46.445827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:51.462085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:56.485822 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:12:58.587225 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 09:13:01.499296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:06.521553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:11.535399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:16.559648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:21.571568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:26.587809 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:31.621291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:36.638700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:41.661475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:46.678224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:48.817830 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 09:13:51.695814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:13:56.717260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:01.735234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:06.751630 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:11.772689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:16.793241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:21.812479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:26.830241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:31.849155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:36.873378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:42.009758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:47.025708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:52.046707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:14:57.135553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:02.150702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:07.168046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:12.186551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:17.206185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:22.223115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:27.105539 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 6 items received I0617 09:15:27.261867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:32.278563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:37.297838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:42.311309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:47.336576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:52.361689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:15:57.389375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:02.407767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:07.423199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:12.439994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:17.462599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:22.479831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:27.503512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:32.536347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:37.557333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:42.584949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:47.602725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:52.641721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:16:57.654168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:02.669869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:07.696287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:12.710977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:14.505115 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:17:14.505273 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 09:17:14.505339 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:17:14.505350 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:17:14.505490 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.505529 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.505579 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:17:14.505617 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:17:14.505674 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:17:14.505692 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:17:14.505760 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:17:14.505772 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:17:14.505781 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:17:14.505965 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:17:14.506007 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:17:14.506061 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:17:14.506152 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 09:17:14.505317 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 09:17:14.506608 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:17:14.506752 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:17:14.506795 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.506861 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.506932 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:17:14.505904 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:17:14.507040 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 09:17:14.507056 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 09:17:14.507066 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 09:17:14.507138 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 09:17:14.507148 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 09:17:14.507153 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 09:17:14.507160 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 09:17:14.507167 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 09:17:14.507173 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 09:17:14.507210 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 09:17:14.507218 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 09:17:14.507226 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 09:17:14.507230 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 09:17:14.507236 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 09:17:14.507260 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 09:17:14.507321 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 09:17:14.507394 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:17:14.507409 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:17:14.507418 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:17:14.507519 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:17:14.507532 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:17:14.507593 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:17:14.507646 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:17:14.507677 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:17:14.507686 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 09:17:14.507767 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:17:14.507779 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:17:14.507832 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.507843 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.507850 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:17:14.507857 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:17:14.507863 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:17:14.508313 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:17:14.508336 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:17:14.508342 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:17:14.508348 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:17:14.508388 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:17:14.508396 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:17:14.508402 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:17:14.508412 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 09:17:14.579977 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:17:14.580104 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:17:14.580206 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:17:14.580292 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:17:14.580777 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:17:14.580852 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:17:14.580416 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 09:17:14.580980 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:17:14.580995 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:17:14.581037 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.581062 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:17:14.582151 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:17:14.582173 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:17:14.582181 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:17:14.582196 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:17:14.582274 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:17:14.582286 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:17:14.582296 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:17:14.582391 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:17:14.582447 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:17:14.582458 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 09:17:14.639151 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:17:14.639263 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:17:14.639289 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:17:14.639331 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:17:14.639340 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:17:14.639353 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:17:14.639388 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 09:17:17.726002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:22.739583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:27.756134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:32.779188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:37.799975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:42.826149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:45.655614 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 09:17:47.841311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:52.895529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:17:57.915937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:02.932753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:07.982124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:12.997374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:18.019723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:23.038237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:28.065666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:28.591028 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 09:18:33.081741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:38.097131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:43.118532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:48.139336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:53.164181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:18:58.186419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:03.216983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:08.236860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:13.305742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:18.329863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:23.373440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:28.395669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:33.417906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:38.443509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:43.462924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:48.478103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:53.548674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:19:58.563957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:03.584983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:08.599895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:13.629911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:18.652007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:23.685750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:28.703686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:33.719027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:38.741594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:43.754595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:48.792071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:53.912492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:20:58.926910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:03.943298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:08.990460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:14.011701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:19.028957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:24.071178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:29.088557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:34.102010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:39.127717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:41.842929 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 09:21:44.144461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:49.165593 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:54.196707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:21:59.221778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:04.239953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:09.256443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:14.269801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:19.287438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:24.331490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:29.354914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:34.372463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:39.397652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:44.415316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:49.436081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:54.461257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:22:59.483408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:04.502556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:09.520853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:14.538166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:19.566555 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:24.585965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:29.610073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:34.625345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:39.643745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:44.679812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:49.693857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:54.722932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:23:59.750441 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:04.773184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:07.215136 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 09:24:07.611464 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 09:24:09.790464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:14.806646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:19.820969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:25.260053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:30.295193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:35.312339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:40.330407 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:45.343477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:50.361028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:24:55.380387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:00.402444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:05.434775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:10.456481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:15.476985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:20.551289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:25.581179 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:30.604199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:35.624943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:40.649732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:45.669168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:50.689352 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:25:56.037736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:01.085105 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:06.104590 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:11.125044 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:16.140233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:21.160508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:26.237232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:31.253503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:36.267031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:38.664560 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 09:26:41.283183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:46.296498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:51.317064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:26:56.537277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:01.614357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:06.645978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:11.672274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:13.874922 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 09:27:16.701559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:21.721291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:26.736961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:31.754763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:36.778041 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:41.798374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:46.816742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:51.832480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:27:56.845607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:01.865638 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:06.877433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:11.985203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:17.003841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:22.020326 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:27.310987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:32.322525 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:37.342127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:42.361086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:47.378293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:52.393011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:28:57.409927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:02.429780 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:07.443290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:12.462349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:17.475893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:22.494163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:27.548833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:32.573884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:37.585908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:42.606890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:47.619583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:52.640172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:29:57.845924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:02.876984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:07.892601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:12.920507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:17.938913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:22.959448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:27.989773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:33.006796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:38.024248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:43.041058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:48.061705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:53.078999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:30:58.091631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:03.109241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:08.123436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:10.616032 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 09:31:13.146587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:18.182262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:23.208407 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:28.227592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:33.340818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:38.353835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:43.370539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:48.392082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:53.444278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:31:58.461596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:03.490506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:08.562545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:13.589103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:14.506289 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:32:14.506362 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 09:32:14.506375 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 09:32:14.506383 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:32:14.506388 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:32:14.506394 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.506398 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.506408 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:32:14.506413 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:32:14.506417 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:32:14.506426 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:32:14.506435 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:32:14.506440 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:32:14.506448 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:32:14.506495 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:32:14.506506 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:32:14.506510 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 09:32:14.506873 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:32:14.506879 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:32:14.506885 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.506889 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.506894 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:32:14.506900 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:32:14.506906 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:32:14.506914 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:32:14.506921 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:32:14.506925 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:32:14.506937 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:32:14.506967 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:32:14.506973 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:32:14.506977 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:32:14.506984 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 09:32:14.507041 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:32:14.507062 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 09:32:14.507069 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 09:32:14.507074 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 09:32:14.507080 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 09:32:14.507085 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 09:32:14.507088 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 09:32:14.507092 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 09:32:14.507097 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 09:32:14.507100 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 09:32:14.507109 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 09:32:14.507115 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 09:32:14.507122 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 09:32:14.507125 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 09:32:14.507129 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 09:32:14.507132 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 09:32:14.507135 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 09:32:14.507143 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:32:14.507146 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:32:14.507151 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.507153 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.507157 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:32:14.507162 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:32:14.507166 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:32:14.507170 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:32:14.507175 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:32:14.507177 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:32:14.507181 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:32:14.507205 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:32:14.507215 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:32:14.507219 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:32:14.507223 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 09:32:14.588791 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:32:14.588815 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:32:14.588844 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:32:14.588853 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:32:14.588858 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:32:14.588869 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:32:14.588900 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:32:14.588906 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:32:14.588976 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.588980 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:32:14.588988 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:32:14.589026 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:32:14.589032 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:32:14.589045 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:32:14.589054 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:32:14.589059 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:32:14.589068 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:32:14.589108 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:32:14.589236 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:32:14.589248 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 09:32:14.589574 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 09:32:14.641867 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:32:14.641888 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:32:14.641915 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:32:14.642373 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:32:14.642549 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:32:14.642638 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:32:14.642522 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 09:32:18.605720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:23.627550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:28.644972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:33.679636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:38.694271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:43.722006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:48.745529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:49.223870 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 09:32:53.782620 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:32:58.795983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:03.817840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:08.841929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:13.865197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:18.894131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:24.148953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:29.308615 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:34.598946 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:39.622781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:44.649512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:49.670012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:54.692210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:33:59.712361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:04.730869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:09.749724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:14.766212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:19.784478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:24.833796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:29.852304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:34.871885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:39.891657 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:44.912058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:49.930774 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:34:54.980442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:00.022698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:05.068723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:10.088744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:13.903866 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 09:35:15.105171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:20.137286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:25.175948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:30.196760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:35.219028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:40.236928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:45.258124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:50.275805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:35:55.293380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:00.324266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:02.678526 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 09:36:05.344544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:10.379678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:15.398822 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:20.423765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:25.448326 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:30.461979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:35.483049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:40.503199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:45.522330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:50.541327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:36:56.108549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:01.136499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:06.158496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:11.222119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:16.237463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:21.276186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:26.316660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:31.330760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:36.343915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:41.358129 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:46.372734 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:51.386949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:37:56.692309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:01.735380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:06.761351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:11.783225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:16.800394 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:21.816065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:26.861172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:33.232435 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:38.268130 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:43.289306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:48.315810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:53.407687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:38:58.428652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:01.631743 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 09:39:03.446570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:08.475013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:13.496009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:18.532260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:23.805252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:28.925880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:31.605299 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 09:39:34.092900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:39.117717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:44.136717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:49.156097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:54.315086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:39:59.340058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:04.365367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:09.388536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:14.407175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:19.431673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:24.454859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:29.481858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:34.507831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:39.525184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:44.545016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:49.570194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:54.601882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:40:59.618337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:04.635394 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:09.650661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:14.672010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:19.691851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:26.477358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:31.494208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:36.506954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:41.524049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:46.595754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:51.618137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:41:56.756886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:01.793360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:06.809731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:11.829382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:16.843365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:21.861317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:26.879510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:31.895650 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:36.919190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:41.991120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:47.010020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:52.027482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:42:57.047222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:02.073720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:07.091392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:12.115524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:17.129945 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:22.215747 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:27.235405 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:32.270797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:37.284028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:42.313260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:47.329472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:52.352348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:43:57.565957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:02.594791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:07.614501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:12.635783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:17.647942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:22.666955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:23.961548 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 09:44:27.683595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:32.697069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:37.712953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:42.755894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:47.787915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:52.804906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:44:57.829636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:00.686587 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 09:45:02.849871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:07.867769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:12.890038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:17.906057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:22.932267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:27.957855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:32.989847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:38.005934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:43.024187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:48.038979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:53.055108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:45:58.071985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:03.093310 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:08.118769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:13.141831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:18.166002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:23.192990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:28.209778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:33.240700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:38.265616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:43.285597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:48.299342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:53.369957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:46:58.391346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:03.425004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:08.445777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:13.466455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:14.507306 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:47:14.507512 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 09:47:14.507624 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 09:47:14.507711 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 09:47:14.507765 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 09:47:14.507834 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 09:47:14.507846 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 09:47:14.507850 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 09:47:14.507858 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 09:47:14.507864 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 09:47:14.507869 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 09:47:14.507885 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 09:47:14.507890 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 09:47:14.507895 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 09:47:14.507898 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 09:47:14.507902 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 09:47:14.507908 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 09:47:14.507910 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 09:47:14.507921 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:47:14.507924 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:47:14.507931 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:47:14.507934 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:47:14.507976 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 09:47:14.507976 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.507984 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.507986 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.507988 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.507995 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:47:14.508004 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:47:14.508006 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 09:47:14.508010 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:47:14.507995 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:47:14.508018 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:47:14.508020 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:47:14.508035 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:47:14.508051 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:47:14.508059 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:47:14.508064 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:47:14.508073 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:47:14.508129 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:47:14.508137 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:47:14.508149 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:47:14.508160 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 09:47:14.508181 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 09:47:14.508191 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 09:47:14.508197 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.508202 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.508208 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:47:14.508221 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 09:47:14.508226 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 09:47:14.508232 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 09:47:14.508242 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 09:47:14.508249 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:47:14.508255 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:47:14.508267 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 09:47:14.508275 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 09:47:14.508282 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 09:47:14.508287 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 09:47:14.508029 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:47:14.508299 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:47:14.508305 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:47:14.508340 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:47:14.508355 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:47:14.508360 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 09:47:14.573295 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:47:14.573494 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:47:14.573586 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:47:14.574013 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:47:14.574100 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:47:14.574171 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:47:14.574273 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 09:47:14.574286 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 09:47:14.573970 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 09:47:14.574344 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.574369 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 09:47:14.574430 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 09:47:14.574469 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 09:47:14.574482 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 09:47:14.574558 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 09:47:14.574616 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 09:47:14.574651 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 09:47:14.574736 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 09:47:14.574806 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 09:47:14.574875 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 09:47:14.574915 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 09:47:14.627079 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 09:47:14.627347 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 09:47:14.627461 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 09:47:14.627544 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 09:47:14.627555 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 09:47:14.627566 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 09:47:14.627950 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 09:47:18.483916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:21.624686 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 6 items received I0617 09:47:23.509647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:28.523498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:33.551035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:38.569345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:43.590534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:48.613965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:54.029715 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:47:59.112995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:04.131480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:04.653690 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 09:48:09.184350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:14.211099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:19.227736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:24.262183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:29.279074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:34.299965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:39.320808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:44.335827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:49.355247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:54.374742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:48:59.397607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:04.419923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:09.437739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:14.457823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:19.477189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:24.856367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:29.883577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:34.910887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:39.926597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:44.946783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:49.966258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:49:54.999266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:00.016503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:05.053639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:10.068169 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:15.097458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:20.115442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:25.156732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:30.171592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:35.193142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:40.206753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:45.227321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:50.244703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:50:53.702999 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 09:50:55.275520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:00.300497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:05.322479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:10.339094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:15.359438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:20.381177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:25.415408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:30.435469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:35.451900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:40.470810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:45.486508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:50.505184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:51:55.568707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:00.581843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:05.597058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:10.609094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:15.622938 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:20.633256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:25.654303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:30.670524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:35.687029 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:40.709253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:45.729295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:46.641644 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 1 items received I0617 09:52:50.745963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:52:55.770259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:00.794585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:05.813297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:10.831695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:15.855878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:20.878612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:25.894636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:30.913533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:35.936327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:40.958996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:45.977302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:50.995051 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:53:56.018094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:01.036604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:02.056510 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 09:54:06.055897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:11.082324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:16.101377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:21.117984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:26.331240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:31.348444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:36.373056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:41.392118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:46.410772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:51.441692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:54:56.453554 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:01.478477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:06.493556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:11.507993 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:16.526135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:21.546796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:26.569381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:31.587433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:36.610695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:41.632962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:46.649172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:51.670893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:55:56.728476 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:01.748682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:06.762816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:11.777505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:16.793990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:21.807391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:26.969108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:29.670584 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 09:56:32.000099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:37.031660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:42.043102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:47.060463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:52.076472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:56:57.347080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:02.366827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:07.391948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:12.409959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:17.432828 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:22.450300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:27.472186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:32.488566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:37.512019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:42.527108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:47.547011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:52.565751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:57:57.581161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:02.596191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:07.617197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:12.635096 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:17.665824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:22.750008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:27.767312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:32.784576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:37.838939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:42.853984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:47.872656 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:52.890713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:58:57.904923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:02.929748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:07.950044 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:12.964995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:17.981139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:23.000361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:28.015852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:33.033281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:38.050695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:40.656993 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 09:59:43.067087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:48.091816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:53.148331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:58.165305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 09:59:59.721481 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 10:00:03.180327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:08.199253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:11.067385 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 10:00:13.215995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:18.234502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:23.305877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:28.389887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:33.426959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:38.524865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:43.537497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:48.570838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:53.636646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:00:58.654790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:03.678839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:08.694490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:13.720096 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:18.739719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:23.803634 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:28.822263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:33.839835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:38.856832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:43.883704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:48.901645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:53.921807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:01:58.944109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:03.966902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:08.990826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:14.012613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:14.508428 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:02:14.508428 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:02:14.508526 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 10:02:14.508540 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 10:02:14.508552 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:02:14.508574 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:02:14.508600 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:02:14.508607 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:02:14.508615 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.508620 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.508628 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:02:14.508636 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:02:14.508641 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:02:14.508653 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:02:14.508664 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:02:14.508669 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:02:14.508677 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:02:14.508731 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:02:14.508744 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.508746 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:02:14.508750 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.508752 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 10:02:14.508757 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:02:14.508764 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:02:14.508769 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:02:14.508777 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:02:14.508785 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:02:14.508791 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:02:14.508798 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:02:14.508856 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:02:14.508872 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:02:14.508877 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:02:14.508886 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 10:02:14.508526 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 10:02:14.508930 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 10:02:14.508939 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 10:02:14.508946 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 10:02:14.508954 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 10:02:14.508958 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 10:02:14.508964 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 10:02:14.508992 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 10:02:14.508996 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 10:02:14.509006 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 10:02:14.509012 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 10:02:14.509017 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 10:02:14.509021 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 10:02:14.509026 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 10:02:14.509031 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 10:02:14.509035 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 10:02:14.509061 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:02:14.509066 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:02:14.509070 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.509075 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.509080 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:02:14.509087 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:02:14.509091 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:02:14.509097 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:02:14.509107 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:02:14.509111 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:02:14.509116 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:02:14.509157 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:02:14.509176 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:02:14.509182 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:02:14.509189 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 10:02:14.565750 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:02:14.565786 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:02:14.565813 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:02:14.565821 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:02:14.565828 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:02:14.565853 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:02:14.565885 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:02:14.565949 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:02:14.565964 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.565970 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:02:14.565979 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:02:14.565989 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:02:14.566018 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:02:14.566034 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:02:14.566049 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:02:14.566057 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:02:14.566065 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:02:14.566113 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:02:14.566129 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:02:14.566138 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 10:02:14.566641 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 10:02:14.650449 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:02:14.650495 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:02:14.650522 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:02:14.650538 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:02:14.650545 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:02:14.650572 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:02:14.650870 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 10:02:19.030288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:24.056908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:29.088023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:34.113313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:39.131104 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:44.147256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:49.166500 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:54.293356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:02:59.319303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:04.346389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:09.379528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:14.395362 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:19.414595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:24.433613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:29.452241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:34.466932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:39.488983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:44.501755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:49.534638 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:54.572599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:03:59.668496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:04.691283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:09.721469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:14.739861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:19.773563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:25.006967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:30.027840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:35.043195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:40.064227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:45.083368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:45.693545 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 10:04:50.101690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:04:55.197999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:00.219481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:05.231550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:10.252827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:14.672774 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 10:05:15.264922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:20.302366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:25.320514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:30.357680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:35.371464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:36.748720 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 10:05:40.558403 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:45.578219 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:50.608909 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:05:55.648512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:00.681811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:05.698276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:10.722593 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:15.742483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:20.760667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:25.782295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:30.797711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:35.823255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:40.846939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:45.870075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:50.890341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:06:55.907379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:00.927086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:05.955120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:10.967986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:16.317691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:21.337724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:26.362497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:31.382229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:36.409253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:41.431878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:46.442990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:51.464354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:07:56.484976 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:01.505969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:06.520784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:11.541895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:16.088633 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 10:08:16.556541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:21.584087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:26.641566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:31.661014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:36.673127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:41.697040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:46.713483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:51.732894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:08:57.109122 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:02.126411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:07.140446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:12.230055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:17.249050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:22.266030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:27.279460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:32.295889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:37.311099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:42.329786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:47.344229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:52.361590 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:09:57.375761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:02.413854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:07.427925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:12.443632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:17.458545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:22.473933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:27.489351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:32.503871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:37.517359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:42.535791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:47.550762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:52.571137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:10:57.583614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:02.605234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:07.621672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:12.641581 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:17.661670 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:22.682484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:27.695689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:32.713327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:37.725449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:42.741516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:47.760366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:52.777926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:11:57.793543 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:02.812628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:03.754376 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 10:12:07.826689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:12.845296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:17.860342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:22.879291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:27.895350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:32.914621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:37.931901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:42.959016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:47.982081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:53.012556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:12:58.029019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:03.053323 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:07.698086 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 10:13:08.069273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:13.083012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:18.098085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:23.116677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:28.134142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:33.152934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:38.167213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:43.412001 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:48.431146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:53.453787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:58.485956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:13:59.697771 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 1 items received I0617 10:14:03.503816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:08.524883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:13.541079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:18.564409 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:23.644721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:28.662129 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:33.702225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:38.720019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:43.743090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:48.763773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:53.977588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:14:58.995485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:04.012769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:09.034296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:14.053573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:19.091094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:24.108453 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:29.129471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:34.148027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:39.165766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:44.184257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:49.203666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:54.221280 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:15:59.257854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:04.271451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:09.291724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:14.339367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:19.359956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:24.388516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:29.412586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:34.431775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:39.452473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:44.470643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:49.485139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:54.615665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:16:59.632756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:04.652081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:06.104842 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 10:17:09.668643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:14.508784 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:17:14.508872 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 10:17:14.508886 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 10:17:14.508897 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:17:14.508903 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:17:14.508911 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.508916 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.508925 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:17:14.508934 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:17:14.508944 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:17:14.508955 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:17:14.508965 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:17:14.508971 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:17:14.508981 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:17:14.509030 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:17:14.509038 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:17:14.509043 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:17:14.509052 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 10:17:14.509076 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:17:14.509082 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:17:14.509087 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.509092 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.509098 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:17:14.509106 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:17:14.509111 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:17:14.509119 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:17:14.509127 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:17:14.508784 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:17:14.509131 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:17:14.509209 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 10:17:14.509220 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 10:17:14.509228 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 10:17:14.509237 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 10:17:14.509246 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 10:17:14.509250 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 10:17:14.509256 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 10:17:14.509262 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 10:17:14.509266 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 10:17:14.509277 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 10:17:14.509286 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 10:17:14.509292 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 10:17:14.509297 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 10:17:14.509301 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 10:17:14.509305 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 10:17:14.509310 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 10:17:14.509321 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:17:14.509327 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:17:14.509332 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.509337 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.509342 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:17:14.509349 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:17:14.509168 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:17:14.509355 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:17:14.509359 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:17:14.509361 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:17:14.509369 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:17:14.509373 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:17:14.509379 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:17:14.509381 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 10:17:14.509384 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:17:14.509446 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:17:14.509453 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:17:14.509458 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:17:14.509464 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 10:17:14.568015 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:17:14.568039 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:17:14.568310 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:17:14.568327 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:17:14.568342 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:17:14.568887 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:17:14.569011 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:17:14.569043 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:17:14.569067 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.569085 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:17:14.569109 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:17:14.569139 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:17:14.569167 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:17:14.569205 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:17:14.569244 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:17:14.569271 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:17:14.568480 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 10:17:14.569302 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:17:14.569461 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:17:14.569478 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:17:14.569483 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 10:17:14.664076 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:17:14.664190 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:17:14.664267 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:17:14.664291 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:17:14.670139 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 10:17:14.670267 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:17:14.670320 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:17:14.693358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:19.716265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:24.731119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:29.742274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:34.781922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:39.793661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:44.820063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:49.832675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:54.861580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:17:59.877752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:04.898124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:09.923248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:14.940992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:19.970177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:25.067429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:30.082557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:35.097140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:36.768475 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 10:18:40.112582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:45.126521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:50.141207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:18:55.162499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:00.176467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:05.189858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:10.204467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:15.224737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:20.243645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:25.263296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:30.287137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:35.302142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:40.338016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:45.352196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:50.371094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:19:55.395334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:00.408819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:05.424970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:10.440058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:15.457998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:20.474259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:25.497998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:30.509713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:35.532053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:40.545233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:45.581065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:50.597448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:20:55.991997 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:01.004987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:06.032944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:11.046783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:16.069270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:21.084729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:26.100182 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:31.125164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:33.721354 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 10:21:36.138914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:41.223850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:46.241147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:47.706836 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 10:21:51.256963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:21:56.273464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:01.292582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:06.309925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:11.327094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:12.108600 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 10:22:16.344617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:21.363324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:26.382551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:31.400302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:36.422521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:41.437744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:46.464233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:51.479449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:22:56.498240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:01.512970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:06.527620 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:11.541135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:16.576806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:21.589076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:26.604095 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:31.619103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:36.635393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:41.650589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:46.665891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:51.681735 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:23:56.908540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:01.925695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:06.959750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:11.997809 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:17.022054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:22.039831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:27.062296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:32.080094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:37.098175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:42.135711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:47.151227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:52.166100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:24:57.183498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:02.201763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:07.219255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:12.239349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:17.256937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:22.268695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:27.541091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:32.561797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:37.579979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:42.626732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:47.644033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:52.670062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:25:57.689735 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:02.705334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:07.721824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:12.756382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:17.776148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:22.798247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:24.782008 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 10:26:27.817133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:32.832033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:37.855585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:42.872286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:47.903483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:52.971763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:26:58.005227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:03.024263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:08.045808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:13.067626 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:18.087824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:23.107668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:28.123037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:33.150732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:38.172677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:43.195204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:48.317832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:53.369588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:27:58.395272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:03.417271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:08.439445 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:13.463904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:18.524879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:23.543720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:28.561277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:33.574144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:38.596065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:43.611991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:48.626254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:53.714249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:28:55.725956 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 10:28:58.730891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:03.744677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:08.775211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:13.788845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:18.809253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:23.973488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:28.988470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:29.121581 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 10:29:34.015066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:39.038508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:44.059941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:49.079028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:54.116754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:29:59.134681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:02.718655 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 10:30:04.151708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:09.169765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:14.189635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:19.209095 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:24.231389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:29.259565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:34.276064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:39.295227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:44.447861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:49.472402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:54.498137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:30:59.517424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:04.537756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:09.553210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:14.566902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:19.588434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:24.603716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:29.625364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:34.652621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:39.669852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:44.685615 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:49.705467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:54.727298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:31:59.743763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:04.757830 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:09.777518 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:14.509540 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:32:14.509803 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:32:14.509699 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 10:32:14.509985 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 10:32:14.510000 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 10:32:14.510034 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 10:32:14.510050 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 10:32:14.510058 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 10:32:14.510065 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 10:32:14.510072 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 10:32:14.510078 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 10:32:14.510128 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 10:32:14.510156 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 10:32:14.510176 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 10:32:14.510225 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 10:32:14.510269 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 10:32:14.510310 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 10:32:14.510343 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 10:32:14.510400 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:32:14.510447 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:32:14.510542 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.510572 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.510632 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:32:14.510691 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:32:14.510400 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:32:14.510767 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:32:14.510774 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.510780 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.510787 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:32:14.510794 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:32:14.510800 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:32:14.510812 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:32:14.510843 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:32:14.510855 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:32:14.510864 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:32:14.510941 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:32:14.510949 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:32:14.510958 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:32:14.510968 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 10:32:14.510999 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:32:14.511026 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:32:14.511080 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:32:14.511115 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:32:14.509962 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 10:32:14.511253 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 10:32:14.511330 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:32:14.511394 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:32:14.511427 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.511476 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.511504 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:32:14.511141 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:32:14.511573 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:32:14.511599 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:32:14.511655 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:32:14.511679 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:32:14.511751 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:32:14.511805 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:32:14.511855 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:32:14.511930 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:32:14.511968 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:32:14.511631 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:32:14.512112 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:32:14.512124 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 10:32:14.512025 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 10:32:14.565299 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:32:14.565482 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:32:14.565563 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:32:14.565590 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:32:14.565641 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:32:14.565676 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:32:14.565756 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:32:14.565829 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:32:14.565891 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.565924 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:32:14.565981 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:32:14.566012 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:32:14.566067 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:32:14.566102 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:32:14.566162 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:32:14.566191 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:32:14.566252 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:32:14.566293 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:32:14.566359 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:32:14.566409 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 10:32:14.566635 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 10:32:14.603651 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:32:14.603782 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:32:14.603822 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:32:14.603865 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:32:14.603886 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:32:14.603917 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:32:14.603980 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 10:32:14.795025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:19.814010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:24.841811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:29.860646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:34.875778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:39.893000 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:44.912885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:49.936342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:54.960072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:32:59.976569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:04.991773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:10.014725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:15.033136 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:20.052923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:25.095208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:30.112499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:31.820120 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 10:33:35.134896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:40.158784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:45.171190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:50.199346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:33:55.294862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:00.324474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:05.337277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:10.354519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:15.369996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:20.388059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:25.405548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:30.421764 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:35.439393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:40.457320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:45.474368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:50.489852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:34:55.503893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:00.521155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:05.542173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:10.559496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:15.581752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:20.594220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:25.614426 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:30.632211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:35.645861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:40.669461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:45.681148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:50.706118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:35:55.721800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:00.751194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:05.765366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:10.783882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:15.800219 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:20.826859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:26.065024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:31.082957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:36.097350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:41.120061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:46.132102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:51.161623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:56.174388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:36:58.726843 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 10:37:01.193837 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:06.209055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:11.232277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:16.247263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:21.271355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:26.366052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:31.387008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:36.406281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:41.429430 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:42.128039 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 10:37:46.452010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:51.470543 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:37:56.622620 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:01.636470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:06.651132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:11.668043 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:16.692650 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:21.711788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:26.736524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:31.755741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:36.780300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:41.803833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:43.757564 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 1 items received I0617 10:38:46.822983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:51.845228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:38:57.039297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:02.061751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:07.073943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:12.095485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:17.115365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:22.145939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:27.210350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:32.230638 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:37.249092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:42.265320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:47.292574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:52.311648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:39:57.334426 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:02.395948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:07.409344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:12.428532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:17.441902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:22.466731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:27.481482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:32.503110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:37.517032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:42.538893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:47.557778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:52.575521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:40:57.590024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:02.613667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:07.635914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:12.655778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:17.677857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:22.705787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:27.722306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:32.750066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:37.768455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:42.785541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:47.799952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:52.824045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:41:57.858931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:02.884115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:07.911039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:12.928257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:17.953622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:22.975434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:28.016233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:33.035734 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:38.062662 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:43.325692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:48.347481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:49.831003 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 10:42:53.446024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:42:58.464137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:03.480254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:08.527890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:13.545274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:18.563215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:23.643857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:28.659528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:33.677547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:38.692354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:41.731424 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 10:43:43.714843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:48.733080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:54.434210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:43:59.455094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:04.477159 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:09.498694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:14.521256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:19.540695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:24.560140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:29.576383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:34.596547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:39.613444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:44.634559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:49.651093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:54.677843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:44:59.696776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:04.715877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:09.734080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:14.754901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:19.771016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:24.809773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:29.830718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:34.849348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:39.864497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:44.881297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:49.900867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:54.965890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:45:59.984072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:05.010760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:10.026082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:15.042179 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:20.058873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:25.087205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:28.130973 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 10:46:30.098991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:35.115963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:35.762991 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 10:46:40.133843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:45.153780 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:50.180795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:46:55.196188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:00.211194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:05.226056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:10.246851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:14.510387 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:47:14.510454 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 10:47:14.510471 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 10:47:14.510486 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 10:47:14.510502 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 10:47:14.510520 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 10:47:14.510526 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 10:47:14.510535 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 10:47:14.510546 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 10:47:14.510550 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 10:47:14.510567 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 10:47:14.510709 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 10:47:14.510630 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 10:47:14.510653 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:47:14.510885 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:47:14.510922 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.510966 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.511009 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:47:14.511060 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:47:14.511100 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:47:14.511148 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:47:14.511186 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:47:14.511231 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:47:14.511264 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:47:14.511371 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:47:14.511457 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:47:14.511486 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:47:14.511510 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 10:47:14.511565 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 10:47:14.511605 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 10:47:14.511642 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:47:14.511670 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:47:14.511702 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.511734 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.511768 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:47:14.511800 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:47:14.511812 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:47:14.511821 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:47:14.511830 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:47:14.511835 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:47:14.511842 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:47:14.511885 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:47:14.511899 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:47:14.511908 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 10:47:14.512275 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 10:47:14.512294 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 10:47:14.512303 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.512308 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.512315 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:47:14.512323 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 10:47:14.512331 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 10:47:14.512342 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 10:47:14.512354 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 10:47:14.512360 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:47:14.512367 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:47:14.512404 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 10:47:14.512416 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 10:47:14.512423 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 10:47:14.512431 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 10:47:14.512471 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 10:47:14.512484 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 10:47:14.512493 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 10:47:14.512503 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 10:47:14.512509 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. E0617 10:47:14.610989 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:47:14.611014 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:47:14.611035 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:47:14.611043 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:47:14.611048 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:47:14.611059 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:47:14.611084 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 10:47:14.611097 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 10:47:14.611106 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.611116 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 10:47:14.611125 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 10:47:14.611141 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 10:47:14.611150 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 10:47:14.611163 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 10:47:14.611174 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 10:47:14.611181 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 10:47:14.611704 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 10:47:14.611190 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 10:47:14.611754 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 10:47:14.611772 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 10:47:14.611781 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 10:47:14.652363 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 10:47:14.652389 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 10:47:14.652409 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 10:47:14.652417 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 10:47:14.652424 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 10:47:14.652433 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 10:47:14.652883 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 10:47:15.265325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:20.296336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:25.385317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:30.407726 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:35.426683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:40.447418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:45.467286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:50.488541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:47:55.505053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:00.522083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:05.540090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:10.560546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:15.580758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:20.598057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:25.642225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:30.659160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:35.685931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:40.705805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:45.726844 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:50.744020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:48:55.768286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:00.782045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:05.871051 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:10.891016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:15.916745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:20.929207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:25.982275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:30.998357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:36.016886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:41.035774 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:46.057779 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:51.081738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:49:56.105053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:01.125673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:06.145748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:11.159675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:16.173801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:21.189462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:26.392034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:31.406840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:36.422450 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:41.434847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:46.457259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:51.476911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:50:56.499868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:01.516429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:06.536461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:11.551669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:16.572714 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:21.627770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:26.644889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:31.663445 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:36.683589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:41.702031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:46.721926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:51.737290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:51:56.800853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:01.819744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:06.843462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:11.864913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:16.901192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:21.923624 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:26.941831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:31.958605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:36.974923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:41.844877 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 10:52:41.993045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:47.009383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:52.040270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:52:54.758296 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 10 items received I0617 10:52:57.068651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:02.087150 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:07.114662 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:12.134396 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:17.149829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:22.169754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:27.218119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:32.234245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:37.256992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:42.273302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:47.292502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:52.411841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:53:54.142909 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 10:53:57.427055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:01.779965 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 10:54:02.440771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:07.454090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:12.471312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:17.498099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:22.515560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:27.537449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:32.556399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:37.604768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:42.619101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:47.638990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:52.675240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:54:57.689832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:02.703499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:07.728025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:12.744088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:17.759556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:22.772346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:27.794793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:32.809111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:37.824068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:42.841487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:47.857296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:52.875338 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:55:57.912670 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:02.941214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:08.003601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:13.020877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:18.043039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:23.058810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:28.080424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:33.092848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:38.114950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:43.142782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:48.248954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:53.371769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:56:58.399437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:03.415742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:08.439973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:13.454845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:18.476741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:23.531546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:28.549757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:33.582547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:38.599278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:43.619492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:48.638784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:54.020262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:57:59.036627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:04.064458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:09.090743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:14.147067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:19.164319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:24.457748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:29.476721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:34.494157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:39.524695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:44.541808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:49.562889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:54.595913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:58:59.611104 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:04.624763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:09.641652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:14.658251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:19.676983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:24.693562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:29.711684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:34.723889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:39.740283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:44.753091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:47.782690 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 10:59:49.766325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:54.781240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 10:59:59.795750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:04.812573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:09.827375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:14.839833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:19.855897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:25.005985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:30.024259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:35.047258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:40.075792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:45.117955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:50.189890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:00:55.382511 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:00.395753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:05.413933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:10.436490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:15.452162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:20.469716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:25.494359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:30.511068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:35.540059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:40.554420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:45.581939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:50.598664 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:01:50.857626 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 11:01:55.617088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:00.633300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:05.650366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:10.675146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:14.510964 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:02:14.511131 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 11:02:14.511207 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 11:02:14.511223 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 11:02:14.511236 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 11:02:14.511246 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 11:02:14.511256 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 11:02:14.511297 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 11:02:14.511309 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 11:02:14.511316 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 11:02:14.511338 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 11:02:14.511345 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 11:02:14.511393 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 11:02:14.511402 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 11:02:14.511408 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 11:02:14.511415 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 11:02:14.511420 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 11:02:14.511469 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:02:14.511480 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:02:14.511499 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.511505 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.511514 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:02:14.511556 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:02:14.511592 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:02:14.511662 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:02:14.511810 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 11:02:14.511827 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 11:02:14.511842 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:02:14.511567 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:02:14.511911 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:02:14.511962 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.512016 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.512073 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:02:14.512147 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:02:14.512181 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:02:14.512248 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:02:14.512265 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:02:14.512307 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:02:14.512001 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:02:14.512396 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:02:14.512418 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:02:14.512514 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:02:14.512555 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:02:14.512567 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:02:14.512636 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 11:02:14.512337 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:02:14.512715 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:02:14.512771 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:02:14.512819 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 11:02:14.512803 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:02:14.513049 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:02:14.513100 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.513131 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.513155 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:02:14.513204 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:02:14.513240 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:02:14.513265 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:02:14.513293 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:02:14.513330 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:02:14.513358 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:02:14.513420 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:02:14.513441 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:02:14.513458 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:02:14.513477 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 11:02:14.573843 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:02:14.574037 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:02:14.574105 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:02:14.574137 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:02:14.574310 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:02:14.574699 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:02:14.574472 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:02:14.575501 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:02:14.575546 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:02:14.575582 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.575603 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:02:14.575633 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:02:14.575660 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:02:14.575695 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:02:14.575728 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:02:14.575779 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:02:14.575803 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:02:14.575830 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:02:14.575894 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:02:14.575938 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:02:14.575971 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 11:02:14.616975 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:02:14.616997 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:02:14.617056 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:02:14.617087 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:02:14.617199 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:02:14.617215 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:02:14.617464 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:02:15.691446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:18.155956 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 11:02:20.708961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:25.723704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:30.750717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:35.776584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:38.787901 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 11:02:40.793876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:45.869713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:50.885747 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:02:56.120682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:01.136788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:06.152132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:11.175501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:16.187466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:21.201157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:26.214266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:31.232832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:36.251989 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:41.266340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:46.380127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:51.396584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:03:56.410354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:01.452303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:06.466515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:11.488841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:16.501919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:21.517490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:26.537906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:31.559324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:36.579699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:41.594683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:46.623795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:51.646070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:04:56.667576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:01.687262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:06.711032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:11.725128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:16.742076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:21.794872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:27.045812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:32.063707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:37.082017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:42.096770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:47.115153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:52.139965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:05:57.389785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:02.406796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:07.428013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:12.445254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:17.464232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:22.481392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:27.503993 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:32.522788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:37.541276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:42.582185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:47.597164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:52.627257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:06:57.642050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:02.659142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:04.796348 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 11:07:07.676113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:12.698089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:17.720875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:22.735297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:27.748674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:32.766561 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:37.782205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:42.801365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:47.817622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:52.837667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:57.851189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:07:57.875891 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 11:08:02.869548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:07.886038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:12.903748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:17.953461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:22.981117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:28.014428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:33.030772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:38.045980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:43.062490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:48.077300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:53.094854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:08:58.111544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:03.136754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:08.155039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:13.194050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:18.212732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:21.797478 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 1 items received I0617 11:09:23.318680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:28.343942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:33.368213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:38.384467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:43.401949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:48.432253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:53.455949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:09:58.472085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:03.511953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:08.543183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:09.169061 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 11:10:13.562528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:18.587107 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:23.662164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:28.681701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:33.713205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:38.732366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:43.752583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:48.768739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:53.969139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:10:58.987557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:04.038755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:09.058121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:14.076746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:19.095225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:24.124338 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:29.248810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:34.308026 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:39.323348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:44.355489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:49.373738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:54.395014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:11:59.412248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:04.429229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:09.450118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:14.469762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:19.487620 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:24.514716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:29.531047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:34.551708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:39.572998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:44.593980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:49.615924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:54.647258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:12:59.663259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:04.687351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:09.703301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:14.722173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:19.743355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:24.762284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:26.941562 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 11:13:29.777465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:34.800480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:39.817072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:44.835930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:49.852448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:54.916416 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:13:59.935308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:04.952066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:09.977558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:14.992351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:20.019455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:25.354691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:30.372478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:35.391077 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:40.406143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:45.425609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:50.441682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:14:55.462010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:00.478367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:05.497039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:10.513006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:15.533292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:20.548170 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:25.577849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:30.609494 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:35.627004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:40.643551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:45.659948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:47.806786 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 11:15:50.673911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:15:55.929752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:00.948770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:05.986801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:09.810603 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 11:16:11.005019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:16.023355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:21.044364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:26.063716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:31.081803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:36.100757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:41.123096 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:46.146455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:51.163761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:16:56.181391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:01.198686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:06.216147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:11.239311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:14.511417 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:17:14.511478 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 11:17:14.511490 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 11:17:14.511497 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 11:17:14.511506 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 11:17:14.511514 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 11:17:14.511517 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 11:17:14.511523 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 11:17:14.511528 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 11:17:14.511531 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 11:17:14.511545 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 11:17:14.511549 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 11:17:14.511553 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 11:17:14.511556 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 11:17:14.511559 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 11:17:14.511563 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 11:17:14.511566 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 11:17:14.511574 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:17:14.511578 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:17:14.511583 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.511587 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.511592 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:17:14.511599 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:17:14.511605 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:17:14.511611 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:17:14.511618 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:17:14.511621 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:17:14.511627 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:17:14.511672 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:17:14.511682 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:17:14.511685 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 11:17:14.512527 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:17:14.512536 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:17:14.512542 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.512547 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.512554 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:17:14.512560 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:17:14.512564 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:17:14.512573 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:17:14.512580 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:17:14.512587 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:17:14.512593 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:17:14.512621 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:17:14.512627 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:17:14.512631 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:17:14.512638 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 11:17:14.512836 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:17:14.512921 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 11:17:14.512948 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 11:17:14.512954 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:17:14.513016 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:17:14.513037 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.513054 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.513093 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:17:14.513122 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:17:14.513160 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:17:14.513194 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:17:14.513234 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:17:14.513264 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:17:14.513284 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:17:14.513447 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:17:14.513497 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:17:14.513554 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:17:14.513597 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 11:17:14.569217 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:17:14.569514 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:17:14.569687 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:17:14.570263 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:17:14.570427 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:17:14.570549 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:17:14.570699 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:17:14.570828 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:17:14.569809 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:17:14.570978 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.571103 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:17:14.571205 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:17:14.571296 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:17:14.571371 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:17:14.571454 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:17:14.571534 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:17:14.571613 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:17:14.571748 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:17:14.571840 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:17:14.571929 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:17:14.572006 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 11:17:14.612309 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:17:14.612335 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:17:14.612361 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:17:14.612371 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:17:14.612376 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:17:14.612390 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:17:14.612430 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:17:16.258149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:21.272519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:26.293834 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:31.318245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:36.335455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:41.353617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:46.372180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:51.391056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:17:56.408756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:01.438487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:06.502101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:11.538875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:16.559704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:21.582182 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:26.603051 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:30.175129 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 11:18:31.619101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:36.635692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:41.648970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:46.665138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:51.679126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:18:56.694248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:01.711168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:06.725218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:11.739166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:16.756190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:21.775538 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:26.792385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:31.810460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:36.824114 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:41.841222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:46.859850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:51.873067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:19:56.890006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:01.905365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:06.923127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:12.078350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:17.092234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:22.110456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:27.127466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:32.145253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:37.163510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:42.177478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:47.192399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:52.208861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:20:57.225911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:02.262055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:07.285798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:12.304752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:17.323462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:22.338485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:26.045571 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 11:21:27.352482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:32.405820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:37.423066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:42.436870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:47.452999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:52.467296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:21:57.482863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:02.500352 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:07.517872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:12.529816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:17.544814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:22.562457 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:27.575680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:32.603280 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:37.617073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:42.643937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:47.659681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:52.674920 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:22:57.695287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:02.711168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:07.747287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:12.763438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:17.781648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:22.795522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:27.810622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:32.824514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:37.840722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:42.855624 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:47.870184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:48.817036 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 11:23:52.897763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:23:57.913865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:02.927584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:07.944879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:12.872291 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 9 items received I0617 11:24:12.961962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:17.978656 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:22.994213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:28.028322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:33.043354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:38.096158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:43.117874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:48.135126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:53.162078 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:24:58.184951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:03.203141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:08.219681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:13.242336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:18.262123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:23.345899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:28.367892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:33.387727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:38.410320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:43.436393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:48.513792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:53.548251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:25:58.569492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:03.597953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:08.619400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:13.639554 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:18.659765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:23.695538 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:28.718253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:33.754235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:38.776736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:43.798827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:48.815690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:50.193558 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 11:26:53.845925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:26:58.890529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:03.914385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:08.945469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:13.962342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:18.055047 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 11:27:18.989356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:24.289932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:29.317023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:34.337307 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:39.397655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:44.419287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:49.449418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:54.472717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:27:59.490748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:04.508075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:09.528021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:14.543309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:19.568665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:24.588374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:29.609113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:34.632011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:39.659880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:44.680780 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:49.712888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:54.740448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:28:59.762364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:04.783072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:09.812289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:14.829857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:19.850611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:24.878145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:29.899884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:34.920052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:39.945710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:44.963241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:49.984013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:29:55.003840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:00.038124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:05.060566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:10.082877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:15.099479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:20.120031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:25.303285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:30.320021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:35.339934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:40.366330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:45.385447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:50.412596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:30:55.428036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:00.459092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:05.478456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:10.506968 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:15.524151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:20.549814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:25.612186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:30.632857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:35.650647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:40.677198 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:44.894201 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 11:31:45.693534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:50.714318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:31:55.896494 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:00.929056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:05.944116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:10.964187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:14.512049 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:32:14.512123 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 11:32:14.512141 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 11:32:14.512158 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 11:32:14.512172 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 11:32:14.512186 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 11:32:14.512192 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 11:32:14.512199 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 11:32:14.512211 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 11:32:14.512220 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 11:32:14.512243 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 11:32:14.512251 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 11:32:14.512258 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 11:32:14.512263 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 11:32:14.512278 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 11:32:14.512288 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 11:32:14.512293 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 11:32:14.512306 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:32:14.512315 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:32:14.512324 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.512333 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.512342 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:32:14.512355 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:32:14.512362 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:32:14.512373 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:32:14.512388 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:32:14.512394 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:32:14.512404 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:32:14.512447 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:32:14.512458 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:32:14.512468 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:32:14.512479 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 11:32:14.512496 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:32:14.512502 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:32:14.512508 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.512513 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.512519 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:32:14.512527 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:32:14.512537 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:32:14.512545 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:32:14.512557 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:32:14.512562 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:32:14.512567 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:32:14.512586 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:32:14.512601 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:32:14.512610 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 11:32:14.513033 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:32:14.513078 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 11:32:14.513093 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 11:32:14.513103 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:32:14.513112 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:32:14.513117 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.513122 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.513128 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:32:14.513137 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:32:14.513143 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:32:14.513153 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:32:14.513162 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:32:14.513171 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:32:14.513179 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:32:14.513214 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:32:14.513228 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:32:14.513236 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:32:14.513245 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 11:32:14.602646 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:32:14.602765 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:32:14.602845 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:32:14.603375 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:32:14.603404 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:32:14.603417 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:32:14.603502 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:32:14.603543 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:32:14.603609 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.603681 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:32:14.603032 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:32:14.603815 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:32:14.603856 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:32:14.603930 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:32:14.603973 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:32:14.604027 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:32:14.604039 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:32:14.604048 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:32:14.604103 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:32:14.604141 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:32:14.604163 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 11:32:14.654211 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:32:14.654237 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:32:14.654256 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:32:14.654265 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:32:14.654271 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:32:14.654283 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:32:14.654793 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:32:15.979938 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:21.001874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:26.382975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:31.410944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:36.430199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:41.452061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:46.475699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:51.501895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:32:56.243791 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 11:32:56.525923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:01.561829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:06.584480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:11.603391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:16.627698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:21.648039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:23.067503 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 11:33:24.830895 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 11:33:26.663411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:31.684257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:36.708020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:41.727908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:46.813300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:51.835171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:33:56.858603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:01.875931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:06.896918 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:11.919249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:16.939656 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:21.973850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:27.275374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:32.311759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:37.327189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:42.350904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:47.367108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:52.390218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:34:57.420369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:02.443179 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:07.471818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:12.487665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:17.506881 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:22.530036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:27.548246 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:32.563839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:37.579731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:42.597260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:47.616437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:52.648096 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:35:57.798433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:02.813805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:07.837343 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:12.853762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:17.876189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:22.893250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:28.130683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:33.182741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:38.210049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:43.432592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:48.448761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:53.503604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:36:58.518397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:03.544744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:08.559495 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:13.578776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:18.598144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:23.660005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:28.677301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:33.704856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:38.727605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:43.758676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:48.777071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:53.893957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:37:58.912951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:04.080059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:09.097452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:14.118484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:19.137128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:24.197868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:25.077450 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 11:38:29.217497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:34.237812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:39.257346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:44.272761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:49.293116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:54.340886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:38:56.997210 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 11:38:59.355273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:04.373247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:09.389365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:14.403102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:19.421812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:24.439862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:29.454769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:34.479122 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:39.509038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:44.525658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:49.542928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:54.564527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:39:59.583873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:04.598783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:09.631166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:14.650038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:19.669609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:24.686592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:29.705569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:34.727149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:35.835193 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 11:40:39.746287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:44.767594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:49.785503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:54.813768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:40:59.825332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:04.845908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:09.871950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:14.939671 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:19.973550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:25.265197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:30.284413 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:35.307722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:39.376089 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 2 items received I0617 11:41:40.324628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:45.342719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:50.364318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:41:55.381532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:00.401771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:05.421905 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:10.442097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:15.461177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:20.485006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:25.649037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:30.663063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:35.685193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:40.702827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:45.718449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:50.739775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:42:55.765770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:00.782887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:05.803455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:10.821344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:15.843303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:20.866038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:25.884559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:30.904393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:35.926107 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:40.949049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:45.991008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:51.008501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:43:56.319007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:01.338232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:06.357083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:11.377954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:16.407624 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:21.425474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:26.446642 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:31.463807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:36.489351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:41.510169 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:46.530264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:51.548297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:44:56.565785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:01.584893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:06.600228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:11.613686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:16.639253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:21.654431 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:26.679694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:31.705517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:36.725003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:41.748498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:45.009932 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 11:45:46.770635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:51.796731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:45:56.831819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:01.880284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:06.900358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:12.001341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:17.017613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:22.033949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:27.049187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:32.063741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:37.079639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:42.092847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:47.108949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:52.125732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:46:57.313997 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:02.339084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:07.362614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:12.383668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:14.512926 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:47:14.513128 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 11:47:14.513225 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 11:47:14.513242 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 11:47:14.513256 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:47:14.513262 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:47:14.513272 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.513288 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.513295 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:47:14.513303 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:47:14.513310 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:47:14.513322 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:47:14.513334 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:47:14.513339 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:47:14.513347 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:47:14.513453 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:47:14.513463 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:47:14.513470 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:47:14.513481 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 11:47:14.513501 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:47:14.513516 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:47:14.513522 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.513526 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.513532 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:47:14.513539 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:47:14.513545 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:47:14.513554 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:47:14.513563 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:47:14.513568 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:47:14.513575 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:47:14.513591 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:47:14.513605 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:47:14.513612 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 11:47:14.514065 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 11:47:14.514094 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 11:47:14.514105 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 11:47:14.514115 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 11:47:14.514126 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 11:47:14.514134 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 11:47:14.514143 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 11:47:14.514151 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 11:47:14.514156 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 11:47:14.514168 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 11:47:14.514175 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 11:47:14.514180 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 11:47:14.514184 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 11:47:14.514189 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 11:47:14.514193 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 11:47:14.514197 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 11:47:14.514210 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 11:47:14.514215 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 11:47:14.514222 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.514226 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.514233 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:47:14.514241 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 11:47:14.514246 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 11:47:14.514285 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 11:47:14.514295 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 11:47:14.514302 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:47:14.514309 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:47:14.514353 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 11:47:14.514364 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 11:47:14.514370 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 11:47:14.514378 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 11:47:14.581547 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:47:14.581589 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:47:14.581621 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:47:14.581630 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:47:14.581637 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:47:14.582113 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:47:14.582164 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:47:14.582681 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 11:47:14.582699 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 11:47:14.582709 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.582714 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 11:47:14.582723 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 11:47:14.582747 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 11:47:14.582841 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 11:47:14.582890 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 11:47:14.582948 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 11:47:14.582995 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 11:47:14.583036 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 11:47:14.583092 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 11:47:14.583123 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 11:47:14.583147 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 11:47:14.621984 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 11:47:14.622070 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 11:47:14.622117 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 11:47:14.622138 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 11:47:14.622156 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 11:47:14.622177 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 11:47:14.622572 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 11:47:17.403806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:22.471424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:23.087058 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 8 items received I0617 11:47:27.490288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:32.515695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:37.544728 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:42.562230 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:47.576935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:52.592673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:47:57.605931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:02.653965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:07.678640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:12.700505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:17.722625 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:22.751718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:27.775046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:32.792669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:37.813231 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:42.827276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:47.844764 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:52.861793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:48:56.393594 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 11:48:57.876682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:02.894209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:07.914382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:12.932185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:17.963281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:22.982986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:28.005367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:33.020605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:38.046877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:43.068273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:48.091210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:53.113632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:49:58.140085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:03.156550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:08.191683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:13.212641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:18.237343 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:23.322364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:28.347949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:29.843347 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 11:50:33.380374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:38.405973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:43.439529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:48.460136 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:53.497452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:50:58.521858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:03.545771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:08.569357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:13.594953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:18.626083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:23.706075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:28.734014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:33.755378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:38.776621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:43.807619 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:48.831456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:53.913273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:51:58.933649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:03.965082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:08.990868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:14.021178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:19.045072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:24.081323 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:29.109744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:34.130694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:39.158646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:44.179035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:49.210781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:54.228420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:52:59.251646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:04.270704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:09.294856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:14.318485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:19.340069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:24.359194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:29.480951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:34.496195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:39.512053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:44.525697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:49.544523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:54.579643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:53:59.599610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:04.627080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:09.647572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:14.668861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:19.689442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:24.715531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:29.742574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:34.763967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:39.800965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:40.034405 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 11:54:44.821183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:49.417167 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 11:54:49.840632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:54.917736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:54:59.931184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:05.027240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:10.047572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:15.060397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:20.079264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:25.093748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:30.114835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:35.130824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:40.149390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:45.164977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:50.193047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:55:55.238008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:00.251934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:05.265137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:10.300498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:15.319568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:20.336070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:25.383497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:30.404271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:35.426184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:36.100381 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 11:56:40.505175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:45.522724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:50.540890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:56:55.935398 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:00.961895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:05.977788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:11.000309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:16.015793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:21.035066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:26.068620 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:31.091213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:36.111458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:41.129964 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:46.153124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:51.170368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:57:56.205027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:01.223423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:06.246652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:11.266409 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:16.292719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:21.310242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:26.360557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:31.376280 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:36.397220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:41.417089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:46.449580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:51.464389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:58:56.797629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:01.820594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:06.839666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:11.860058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:16.879270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:21.910854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:26.931998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:32.000319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:37.033188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:42.054141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:46.850113 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 1 items received I0617 11:59:47.070900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:52.093278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 11:59:55.077606 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 11:59:57.111971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:02.131720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:07.145962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:12.174429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:17.196168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:22.214310 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:27.253939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:32.277865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:37.293550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:42.312270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:47.333535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:52.354398 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:00:57.495338 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:02.513651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:07.532549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:12.547606 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:17.567379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:22.590247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:27.602032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:32.632679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:37.655371 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:42.675520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:47.695300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:52.726381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:01:57.747632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:02.779772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:07.796263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:12.815395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:14.514069 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:02:14.514161 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 12:02:14.514175 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 12:02:14.514183 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 12:02:14.514187 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 12:02:14.514195 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.514200 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.514208 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:02:14.514215 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 12:02:14.514219 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 12:02:14.514229 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 12:02:14.514236 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 12:02:14.514240 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:02:14.514246 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:02:14.514289 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 12:02:14.514297 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 12:02:14.514302 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 12:02:14.514310 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 12:02:14.514319 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:02:14.514322 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:02:14.514327 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.514329 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.514333 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:02:14.514337 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:02:14.514340 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:02:14.514345 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:02:14.514350 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:02:14.514361 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:02:14.514365 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:02:14.514377 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:02:14.514386 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:02:14.514389 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 12:02:14.514907 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:02:14.514944 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 12:02:14.514951 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 12:02:14.514958 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 12:02:14.514967 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 12:02:14.514977 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 12:02:14.514983 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 12:02:14.514987 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 12:02:14.515001 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 12:02:14.515009 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 12:02:14.515013 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 12:02:14.515017 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 12:02:14.515022 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 12:02:14.515042 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 12:02:14.514989 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 12:02:14.515067 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 12:02:14.515072 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 12:02:14.515090 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 12:02:14.515097 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 12:02:14.515104 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.515109 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.515116 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:02:14.515125 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 12:02:14.515132 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 12:02:14.515141 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 12:02:14.515205 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 12:02:14.515263 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:02:14.515283 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:02:14.515382 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 12:02:14.515416 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 12:02:14.515440 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 12:02:14.515500 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 12:02:14.594989 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:02:14.595012 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:02:14.595038 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:02:14.595047 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:02:14.595052 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:02:14.595063 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:02:14.595096 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:02:14.595137 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:02:14.595158 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.595176 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:02:14.595194 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:02:14.595216 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:02:14.595234 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:02:14.595260 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:02:14.595282 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:02:14.595300 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:02:14.595320 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:02:14.595358 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:02:14.595396 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:02:14.595417 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 12:02:14.595781 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 12:02:14.641730 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:02:14.641930 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:02:14.642032 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:02:14.642101 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:02:14.642447 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:02:14.642513 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:02:14.642270 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 12:02:17.855556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:22.875493 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:27.900529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:33.055587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:38.089589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:43.274197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:48.293679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:53.380458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:02:58.398974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:03.425519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:08.448973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:13.468663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:18.499545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:23.589813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:28.614903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:33.632821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:38.648298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:43.665665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:48.697876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:54.843979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:03:59.858116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:04.876709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:05.445201 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 12:04:09.892024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:14.907035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:19.921589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:24.977488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:29.989023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:35.006949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:40.026422 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:45.057455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:50.076025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:04:55.100017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:00.125794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:05.146022 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:10.159034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:15.196315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:20.211191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:25.324100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:30.339842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:35.354368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:40.368148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:45.386226 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:50.409297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:55.438795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:05:59.082956 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 12:06:00.458243 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:05.479633 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:10.501857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:15.518014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:18.119883 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 12:06:20.540299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:25.576985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:30.596057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:35.609975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:40.628426 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:45.663848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:50.815176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:06:55.855232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:00.873005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:05.901069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:10.917386 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:15.934890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:20.957681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:26.005984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:31.024874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:36.058084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:41.076174 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:46.103348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:46.863368 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 12:07:51.119631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:07:56.142970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:01.158009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:06.183468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:11.221745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:16.244412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:21.269598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:26.319614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:31.348977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:36.366135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:41.385286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:46.409463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:51.428935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:08:56.460629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:01.481999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:06.506626 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:11.522979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:16.541264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:21.558021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:26.583814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:27.460822 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 12:09:31.600222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:36.628288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:41.642087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:46.743999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:51.756045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:09:56.986424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:02.013761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:07.054317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:12.070156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:17.087491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:22.100293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:27.168802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:32.185138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:37.204444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:42.220188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:47.239065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:52.256250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:10:57.277066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:02.295524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:07.321371 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:12.337885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:17.352460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:22.638125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:27.654969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:28.090037 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 12:11:32.676015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:37.692550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:42.706249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:47.722108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:52.743959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:11:57.764519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:02.785522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:07.804637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:12.818879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:17.837116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:22.852550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:27.877429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:32.893509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:37.914273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:42.933745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:47.968875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:53.067996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:12:58.095531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:03.122108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:08.160520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:13.220065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:18.246525 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:23.297197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:28.316237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:33.332281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:38.362274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:43.384098 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:48.554594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:53.662789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:13:58.686458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:03.709128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:08.728367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:13.747353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:18.765382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:23.867584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:28.892360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:33.932207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:38.126034 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 12:14:38.947872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:43.962771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:46.497650 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 12:14:48.985986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:54.037163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:14:59.071002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:04.091229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:09.110452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:14.131552 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:19.165867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:24.962041 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:29.985350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:35.003282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:40.020497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:45.033341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:50.050898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:15:55.074483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:00.089146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:05.105628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:10.125607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:15.144424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:20.172613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:25.206825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:30.224360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:35.241077 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:40.263676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:45.299491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:47.117893 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 12:16:50.324529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:16:52.893642 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 12:16:55.387357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:00.417998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:05.432188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:10.448568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:14.515207 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:17:14.515286 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 12:17:14.515298 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 12:17:14.515306 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 12:17:14.515317 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 12:17:14.515324 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 12:17:14.515328 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 12:17:14.515334 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 12:17:14.515339 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 12:17:14.515343 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 12:17:14.515354 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 12:17:14.515358 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 12:17:14.515362 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 12:17:14.515365 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 12:17:14.515369 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 12:17:14.515372 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 12:17:14.515376 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 12:17:14.515384 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 12:17:14.515388 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 12:17:14.515394 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.515399 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.515404 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:17:14.515409 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 12:17:14.515413 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 12:17:14.515419 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 12:17:14.515425 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 12:17:14.515428 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:17:14.515435 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:17:14.515483 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 12:17:14.515491 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 12:17:14.515495 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 12:17:14.515501 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 12:17:14.515508 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:17:14.515511 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:17:14.515514 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.515517 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.515521 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:17:14.515526 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:17:14.515529 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:17:14.515534 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:17:14.515538 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:17:14.515541 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:17:14.515544 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:17:14.515559 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:17:14.515569 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:17:14.515572 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 12:17:14.516109 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:17:14.516138 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 12:17:14.516144 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 12:17:14.516150 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 12:17:14.516153 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 12:17:14.516158 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.516163 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.516168 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:17:14.516173 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 12:17:14.516177 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 12:17:14.516183 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 12:17:14.516189 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 12:17:14.516195 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:17:14.516201 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:17:14.516227 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 12:17:14.516233 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 12:17:14.516237 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 12:17:14.516243 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 12:17:14.578765 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:17:14.578845 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:17:14.578913 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:17:14.579367 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 12:17:14.579459 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:17:14.579747 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:17:14.579801 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:17:14.579882 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:17:14.579910 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:17:14.579960 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.579984 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:17:14.580087 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:17:14.580111 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:17:14.580157 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:17:14.580190 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:17:14.580259 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:17:14.580312 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:17:14.580341 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:17:14.580441 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:17:14.580491 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:17:14.580519 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 12:17:14.622390 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:17:14.622468 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:17:14.622531 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:17:14.622561 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:17:14.622977 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:17:14.623001 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:17:14.622891 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 12:17:15.464190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:20.480529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:25.525938 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:30.724501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:35.747767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:40.782281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:45.798157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:50.823064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:17:55.855878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:00.882288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:05.918259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:10.940127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:15.996771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:21.076149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:26.097219 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:31.116896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:36.139257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:41.162888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:46.196845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:51.219025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:18:56.334278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:01.353980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:06.396259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:11.416248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:16.433437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:21.447906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:26.736905 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:31.764562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:36.791668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:41.820919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:46.844467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:48.507038 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 12:19:51.871759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:19:56.889274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:01.940088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:04.141784 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 12:20:06.954217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:11.989994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:17.009158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:22.029450 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:27.153529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:32.183452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:37.197846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:42.217998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:47.235789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:52.260171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:20:57.274741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:02.296998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:07.319408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:12.337907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:17.366073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:22.384470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:27.405330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:32.501879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:37.518123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:42.536545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:47.552404 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:52.573870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:21:57.602717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:02.629484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:03.921191 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 12:22:07.661462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:12.681164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:17.698375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:22.719197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:27.739184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:32.763661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:37.805760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:42.820558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:47.836115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:52.851154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:22:57.866478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:02.880814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:07.895902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:12.922721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:17.958379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:22.981039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:28.015799 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:33.033712 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:38.047523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:43.065690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:48.082723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:53.102429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:23:58.129765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:03.162075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:08.179650 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:13.205755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:18.227053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:23.275365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:28.296544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:33.320245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:38.337693 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:43.386150 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:48.405669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:53.485266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:24:58.507445 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:03.536369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:08.552606 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:13.576797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:18.594843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:23.671341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:28.696943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:31.124927 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 12:25:33.824047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:38.840402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:43.881050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:48.901220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:53.992319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:25:59.008186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:04.048217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:09.065203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:14.114928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:19.144598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:24.201562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:29.219735 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:34.239667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:39.261846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:44.279838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:49.298825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:54.828974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:26:59.841849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:04.857899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:09.871910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:14.519016 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 12:27:14.889131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:17.148979 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 12:27:19.904605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:24.946875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:29.972479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:34.991690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:40.008607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:45.031468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:50.045900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:27:55.067394 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:00.082976 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:05.115396 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:10.129727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:15.206172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:20.231965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:25.257042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:30.278685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:35.296348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:40.317603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:45.342677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:50.362208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:28:55.385514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:00.398598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:05.419119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:10.437540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:15.455292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:20.473183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:25.494203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:30.527884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:35.542622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:40.564499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:45.589818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:50.611751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:29:55.634929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:00.653202 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:05.679361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:10.694988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:15.714890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:20.733486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:25.953771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:31.125489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:32.140204 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 1 items received I0617 12:30:36.142794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:37.937652 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 1 items received I0617 12:30:41.157708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:46.178584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:51.198156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:30:56.569169 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:01.599204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:06.617813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:11.636055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:16.654715 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:21.673285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:26.689483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:31.711451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:36.730852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:41.743397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:46.763885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:51.776647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:31:56.794872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:01.814456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:06.848875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:11.886801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:14.516288 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:32:14.516360 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 12:32:14.516379 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 12:32:14.516390 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 12:32:14.516401 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 12:32:14.516408 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 12:32:14.516412 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 12:32:14.516419 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 12:32:14.516425 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 12:32:14.516429 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 12:32:14.516439 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 12:32:14.516445 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 12:32:14.516452 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 12:32:14.516456 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 12:32:14.516460 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 12:32:14.516465 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 12:32:14.516470 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 12:32:14.516479 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 12:32:14.516483 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 12:32:14.516488 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.516492 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.516498 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:32:14.516504 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 12:32:14.516508 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 12:32:14.516513 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 12:32:14.516519 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 12:32:14.516523 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:32:14.516530 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:32:14.516585 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 12:32:14.516594 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 12:32:14.516598 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 12:32:14.516605 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 12:32:14.516612 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:32:14.516616 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:32:14.516621 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.516627 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.516632 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:32:14.516638 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:32:14.516642 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:32:14.516646 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:32:14.516671 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:32:14.516675 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:32:14.516679 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:32:14.516688 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:32:14.516699 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:32:14.516704 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 12:32:14.516796 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:32:14.516822 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 12:32:14.516829 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 12:32:14.516835 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 12:32:14.516839 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 12:32:14.516842 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.516847 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.516852 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:32:14.516857 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 12:32:14.516860 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 12:32:14.516866 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 12:32:14.516873 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 12:32:14.516880 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:32:14.516885 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:32:14.516909 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 12:32:14.516915 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 12:32:14.516919 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 12:32:14.516923 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 12:32:14.585751 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:32:14.585802 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:32:14.585832 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:32:14.585862 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:32:14.585906 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:32:14.585938 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:32:14.586011 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:32:14.586034 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:32:14.586072 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.586101 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:32:14.586123 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:32:14.586312 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:32:14.586325 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:32:14.586472 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:32:14.586660 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:32:14.586769 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:32:14.586860 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:32:14.586963 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:32:14.587089 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:32:14.587165 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 12:32:14.586278 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 12:32:14.628212 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:32:14.628328 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:32:14.628395 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:32:14.628420 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:32:14.628440 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:32:14.628487 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:32:14.628610 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 12:32:16.907285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:21.932086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:27.313927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:32.329184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:37.346346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:42.365150 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:47.384019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:52.404439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:32:57.420635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:02.441612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:07.462210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:12.478380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:17.499896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:22.521480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:27.539231 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:32.560420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:37.575878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:42.591299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:47.159817 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 1 items received I0617 12:33:47.606149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:52.627215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:33:57.643075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:02.681887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:07.706089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:12.734301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:17.753174 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:22.765908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:27.780678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:32.797349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:37.817042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:42.831785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:47.848921 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:52.863778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:34:58.125373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:03.149898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:08.171619 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:13.232534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:18.397932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:20.536536 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 12:35:23.735865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:28.758589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:33.775791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:38.804245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:43.832699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:48.859912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:53.900066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:35:58.925803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:03.958333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:09.000257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:14.025049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:19.043675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:24.080149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:29.106205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:34.136645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:39.159198 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:44.180258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:49.199933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:54.340889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:36:59.382474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:04.402713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:09.420354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:14.434985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:19.452907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:24.471765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:29.497368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:34.507713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:39.531774 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:44.546113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:49.567207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:54.649383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:37:59.679814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:04.703101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:09.814162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:14.830478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:19.848746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:24.867100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:29.884996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:34.901616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:39.919622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:44.934728 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:49.961334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:38:54.998603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:00.030317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:05.046325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:10.070953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:15.087508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:20.122904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:23.147752 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 12:39:25.238190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:30.293912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:35.313563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:40.328733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:45.342170 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:50.363644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:39:55.381655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:00.399428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:05.418659 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:10.434991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:15.451227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:20.466275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:24.966745 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 12:40:25.480097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:30.495911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:35.509936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:40.525180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:45.551977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:50.568758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:40:55.602080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:00.616329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:05.680776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:10.697279 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:15.711858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:20.732125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:25.755373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:30.775172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:35.893521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:40.921512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:45.956336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:50.979285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:41:55.998106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:01.024505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:06.046960 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:11.063588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:16.085355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:21.109773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:26.208789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:31.227119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:36.244241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:41.261281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:46.282689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:51.303061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:42:56.376891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:01.397732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:06.415634 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:11.434816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:16.182526 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 12:43:16.458878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:21.481942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:26.499760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:31.518940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:36.536397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:41.552080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:46.566344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:51.589841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:43:56.605133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:01.645210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:06.664086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:07.556807 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 9 items received I0617 12:44:11.683347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:16.709342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:21.727663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:26.769224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:31.791617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:36.809681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:41.833096 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:46.857859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:51.874746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:44:57.112852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:02.134201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:07.152970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:12.177939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:17.207787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:22.231782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:27.266892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:32.304063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:37.321565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:42.343120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:47.361547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:52.389461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:45:57.407347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:02.437524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:07.473507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:12.497216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:17.514216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:22.534614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:27.694030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:32.716218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:37.739421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:42.764782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:47.158183 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 12:46:47.779146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:52.806321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:46:57.821203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:02.838009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:07.851175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:10.050787 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 12:47:12.881046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:14.517007 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:47:14.517112 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 12:47:14.517162 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 12:47:14.517174 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 12:47:14.517191 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 12:47:14.517347 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 12:47:14.517437 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 12:47:14.517461 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 12:47:14.517470 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 12:47:14.517477 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 12:47:14.517484 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 12:47:14.517389 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 12:47:14.517532 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 12:47:14.517547 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 12:47:14.517561 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 12:47:14.517570 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 12:47:14.517576 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:47:14.517609 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 12:47:14.517623 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 12:47:14.517635 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 12:47:14.517643 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 12:47:14.517654 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 12:47:14.517658 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 12:47:14.517690 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:47:14.517703 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 12:47:14.517711 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 12:47:14.517720 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 12:47:14.517732 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 12:47:14.517740 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:47:14.517662 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 12:47:14.517778 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:47:14.517878 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:47:14.517887 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:47:14.517892 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:47:14.517899 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:47:14.517911 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:47:14.517990 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:47:14.518023 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:47:14.518038 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:47:14.518045 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:47:14.518053 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:47:14.518127 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:47:14.518141 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:47:14.518148 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 12:47:14.517811 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:47:14.518699 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 12:47:14.518707 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 12:47:14.518713 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 12:47:14.518723 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 12:47:14.571870 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:47:14.571910 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:47:14.571938 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:47:14.571949 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:47:14.571956 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:47:14.571992 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:47:14.572684 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 12:47:14.572887 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 12:47:14.572904 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 12:47:14.572966 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 12:47:14.572974 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 12:47:14.573042 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 12:47:14.573113 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 12:47:14.573197 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 12:47:14.573283 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 12:47:14.572773 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 12:47:14.573372 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 12:47:14.573389 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 12:47:14.573480 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 12:47:14.573590 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 12:47:14.573603 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 12:47:14.620220 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 12:47:14.620345 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 12:47:14.620415 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 12:47:14.620482 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 12:47:14.620512 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 12:47:14.620538 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 12:47:14.621000 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 12:47:17.900501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:22.918163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:27.973107 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:32.990374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:38.007049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:43.024187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:48.039320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:53.054869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:47:58.071168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:03.086399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:08.100451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:13.130639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:18.157281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:23.182783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:28.207435 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:33.237162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:38.262348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:43.316336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:48.333222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:53.354473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:48:58.389814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:03.409766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:08.432690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:13.465826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:18.502689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:23.550993 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:28.572492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:33.588729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:38.630796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:43.658913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:48.693191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:53.730259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:49:58.751327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:03.769336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:08.788067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:13.810405 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:18.830577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:24.201739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:29.246419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:34.265308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:39.282233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:44.303049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:49.328452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:54.432319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:50:59.454378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:04.478862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:09.495834 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:14.532682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:19.557480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:24.640642 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:29.655471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:34.206464 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 9 items received I0617 12:51:34.677124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:39.694241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:44.718494 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:49.749127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:54.783568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:51:59.808907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:04.830363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:09.851777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:14.870195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:19.889746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:24.963745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:29.989886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:35.010351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:37.566086 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 12:52:40.032340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:45.050663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:50.074697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:52:55.095299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:00.118359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:05.142763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:10.160486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:15.182317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:20.209083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:25.240206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:30.277187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:35.384627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:40.396292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:45.413095 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:50.429858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:53:55.448247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:00.465691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:05.490018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:10.508507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:15.530067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:20.545474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:25.601030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:30.616162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:35.641936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:40.660240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:45.679387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:48.178947 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 12:54:50.709094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:54:56.025832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:01.147729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:06.162042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:10.056924 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 12:55:11.187948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:16.224304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:21.241482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:26.298471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:31.313293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:36.329473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:41.341508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:46.357661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:51.390123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:55:56.462035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:01.503619 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:06.522268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:11.558323 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:16.576118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:21.588615 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:26.607879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:31.625083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:36.672115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:41.687375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:46.719510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:51.741689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:56:56.802502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:01.816585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:06.839659 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:11.862273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:16.885823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:21.906233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:27.153813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:32.171453 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:37.190388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:42.207674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:47.233351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:52.252536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:57:57.297975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:02.327935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:07.350880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:12.369035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:17.386409 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:22.437919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:27.456101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:32.483163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:37.502025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:42.526035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:47.541386 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:52.566062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:58:57.651601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:02.673358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:07.695444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:12.718013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:17.736852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:22.752591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:27.792604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:32.812531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:37.829298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:42.863247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:47.886178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:52.907204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 12:59:57.930114 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:02.947915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:07.973165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:12.986730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:18.011252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:23.029389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:24.212283 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 13:00:28.045545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:30.075313 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 13:00:33.073309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:38.094558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:43.139519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:48.159808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:53.377721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:00:58.394420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:03.409933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:08.443641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:13.470965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:18.514744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:23.557915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:28.575693 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:33.597753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:38.616698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:43.642788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:48.666346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:53.709879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:01:58.739792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:00.572546 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 13:02:03.760672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:08.784093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:13.813109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:14.517941 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:02:14.517941 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:02:14.518029 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 13:02:14.518044 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 13:02:14.518069 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:02:14.518075 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:02:14.518083 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.518089 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.518097 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:02:14.518103 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:02:14.518104 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:02:14.518109 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:02:14.518113 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:02:14.518115 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.518119 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.518124 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:02:14.518124 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:02:14.518131 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:02:14.518137 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:02:14.518145 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:02:14.518150 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:02:14.518154 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:02:14.518157 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:02:14.518029 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 13:02:14.518167 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 13:02:14.518175 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 13:02:14.518181 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 13:02:14.518189 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 13:02:14.518193 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 13:02:14.518199 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 13:02:14.518205 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 13:02:14.518209 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 13:02:14.518227 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 13:02:14.518166 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:02:14.518254 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:02:14.518262 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:02:14.518267 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:02:14.518276 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 13:02:14.518158 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:02:14.518320 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:02:14.518357 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:02:14.518379 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:02:14.518384 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 13:02:14.518234 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 13:02:14.518805 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 13:02:14.518810 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 13:02:14.518819 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 13:02:14.518823 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 13:02:14.518826 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 13:02:14.518859 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:02:14.518864 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:02:14.518869 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.518878 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.518883 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:02:14.518888 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:02:14.518892 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:02:14.518919 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:02:14.518926 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:02:14.518929 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:02:14.518934 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:02:14.518947 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:02:14.518951 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:02:14.518955 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:02:14.518963 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 13:02:14.588970 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:02:14.589057 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:02:14.589104 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:02:14.589136 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:02:14.589153 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:02:14.589180 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:02:14.589221 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:02:14.589239 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:02:14.589268 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.589287 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:02:14.589315 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:02:14.589343 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:02:14.589381 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:02:14.589430 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:02:14.589463 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:02:14.589483 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:02:14.589513 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:02:14.589568 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:02:14.589608 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:02:14.589641 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 13:02:14.590034 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 13:02:14.749386 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:02:14.749514 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:02:14.749594 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:02:14.749625 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:02:14.749797 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:02:14.749852 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:02:14.749762 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 13:02:18.840171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:23.891296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:28.924803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:33.963535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:39.080433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:44.107870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:49.133730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:54.160695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:02:59.190007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:04.208120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:09.233707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:14.255575 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:19.282409 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:24.351876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:29.367913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:34.406266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:39.427195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:44.453962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:49.487690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:54.509541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:03:59.541163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:01.196389 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 13:04:04.562237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:09.583137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:14.600504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:19.622824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:24.639804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:29.661568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:34.679971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:39.700838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:44.716095 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:49.772660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:54.801319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:04:59.824012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:04.836952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:09.849442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:14.864274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:19.882302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:24.988268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:30.010396 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:35.032848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:40.053288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:45.070682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:50.093116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:05:55.120703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:00.141756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:05.158858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:10.182194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:15.198187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:20.224079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:25.510306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:30.527488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:35.541475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:40.561325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:45.581006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:50.610128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:06:55.872616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:00.900663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:05.923539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:10.944500 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:15.962770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:20.990354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:26.007221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:31.027792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:36.044332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:40.252675 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 13:07:41.066702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:43.083553 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 13:07:46.080254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:51.104131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:07:56.125908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:01.153855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:06.169794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:11.203364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:16.226086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:21.245370 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:26.458901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:31.476373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:36.490018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:41.513841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:43.584731 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 13:08:46.533755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:51.555275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:08:56.925760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:02.002987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:07.016689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:12.034901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:17.055052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:22.078008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:27.097130 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:32.118017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:37.133643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:42.149481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:47.167010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:52.188475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:09:57.201851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:02.221438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:07.250139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:12.269623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:17.289462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:22.313645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:27.346454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:32.372977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:37.395655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:42.422574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:47.441222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:52.474952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:10:57.490219 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:02.506216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:07.518564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:12.538769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:17.554125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:22.575786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:27.591351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:32.612850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:37.639904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:42.662006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:47.683015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:52.708766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:11:57.724629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:02.739311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:07.794021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:12.821835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:17.843804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:22.873270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:27.893149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:32.928777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:37.948086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:42.981440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:48.000975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:53.021724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:12:58.041700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:03.065662 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:08.087564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:13.107937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:15.204685 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 13:13:18.130832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:23.297199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:28.321506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:33.357880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:38.389330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:43.410603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:48.483794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:48.726490 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 13:13:53.503763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:13:58.526421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:03.554451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:08.578410 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:13.609726 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:16.090332 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 13:14:18.628078 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:23.784628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:28.804769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:33.824850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:38.841203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:43.855694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:48.873324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:49.260042 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 13:14:54.019006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:14:59.035318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:04.053439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:09.126545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:14.144316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:19.163090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:24.228820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:29.244109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:34.261377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:39.275814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:44.292596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:49.307450 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:54.634991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:15:59.659273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:04.682297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:09.700995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:14.721519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:19.745916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:24.769668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:29.786161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:34.804383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:39.821290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:44.858944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:49.875368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:54.908630 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:16:59.926429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:04.939817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:09.961593 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:14.518918 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:17:14.519265 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 13:17:14.519358 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 13:17:14.519426 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 13:17:14.519472 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 13:17:14.519523 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 13:17:14.519557 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 13:17:14.519586 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 13:17:14.519634 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 13:17:14.519644 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 13:17:14.519675 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 13:17:14.519687 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 13:17:14.519699 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 13:17:14.519704 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 13:17:14.519711 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 13:17:14.519721 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 13:17:14.519765 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 13:17:14.519798 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:17:14.519810 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:17:14.519819 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.519824 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.519869 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:17:14.519895 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:17:14.519967 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:17:14.519982 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:17:14.520006 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:17:14.520029 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:17:14.520075 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:17:14.520157 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:17:14.520196 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:17:14.520207 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 13:17:14.520612 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:17:14.520632 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:17:14.520640 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.520644 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.520687 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:17:14.520709 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:17:14.520748 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:17:14.520787 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:17:14.520825 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:17:14.520865 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:17:14.520903 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:17:14.520975 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:17:14.521019 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:17:14.521055 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:17:14.521098 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 13:17:14.521168 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:17:14.521255 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 13:17:14.521368 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 13:17:14.521434 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:17:14.521456 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:17:14.521490 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.521524 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.521555 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:17:14.521605 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:17:14.521638 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:17:14.521660 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:17:14.521690 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:17:14.521703 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:17:14.521709 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:17:14.521748 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:17:14.521765 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:17:14.521772 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:17:14.521783 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 13:17:14.591048 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:17:14.591073 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:17:14.591102 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:17:14.591110 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:17:14.591116 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:17:14.591128 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:17:14.591164 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:17:14.591194 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:17:14.591214 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.591234 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:17:14.591253 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:17:14.591270 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:17:14.591289 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:17:14.591313 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:17:14.591353 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:17:14.591373 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:17:14.591394 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:17:14.591462 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:17:14.591501 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:17:14.591568 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 13:17:14.592170 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 13:17:14.630898 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:17:14.631174 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:17:14.631237 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:17:14.631644 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:17:14.631656 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:17:14.631668 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:17:14.631404 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 13:17:14.990943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:20.011752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:25.049126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:30.074480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:35.092888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:40.123547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:45.138034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:50.157868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:17:55.173720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:00.186933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:05.206573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:10.229840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:15.263873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:20.284954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:25.307248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:30.331337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:35.349677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:40.367548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:45.388664 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:50.408261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:18:55.425771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:00.451191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:05.465447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:10.484249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:15.503391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:20.529866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:25.653759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:30.667092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:35.683423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:40.698999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:45.720184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:50.745334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:55.764610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:19:58.268946 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 13:20:00.794574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:05.809339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:10.835507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:15.872504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:20.889500 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:26.388341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:29.748746 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 1 items received I0617 13:20:31.406990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:36.428835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:41.445468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:46.463570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:50.095792 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 13:20:51.480336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:20:56.701760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:02.003537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:07.021904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:12.041235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:17.059925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:22.079730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:27.277082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:32.414399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:37.439699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:42.455318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:47.477793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:52.497812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:21:57.520680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:02.554565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:07.589035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:12.608134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:17.628924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:22.645534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:27.667090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:32.682324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:37.706996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:42.719865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:43.209593 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 13:22:47.740173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:52.759678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:22:57.781629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:02.799928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:07.818898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:12.837647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:17.856176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:22.877136 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:27.896927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:32.919725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:37.938491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:42.956551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:47.983225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:53.004571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:23:58.025920 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:03.043446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:08.071663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:13.207901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:18.228968 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:23.512190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:28.535068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:33.553239 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:38.578891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:43.602446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:48.627311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:53.682984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:24:58.707275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:03.723835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:08.753970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:13.773731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:18.800648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:23.919912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:28.945966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:33.964228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:38.992313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:44.010592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:49.045792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:54.166337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:25:59.202687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:04.228755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:09.256007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:14.280325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:19.307074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:24.397801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:29.417716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:34.445355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:39.465823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:44.480754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:49.508221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:54.531417 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:26:59.548166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:04.562049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:09.679485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:14.696730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:19.725167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:24.987740 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:26.754848 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 13:27:30.006328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:35.023336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:36.273603 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 13:27:40.047661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:45.086774 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:50.127702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:27:55.149831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:00.167402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:05.202511 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:10.219953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:15.247284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:20.263465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:25.284076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:30.300079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:35.322012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:40.339898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:45.359887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:50.384109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:28:55.400180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:00.422668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:05.440410 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:10.466720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:15.485841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:20.505995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:25.571547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:30.592989 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:35.616129 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:40.638773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:45.654161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:50.673657 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:29:54.219672 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 13:29:55.105839 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 11 items received I0617 13:29:55.687058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:00.704842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:05.719362 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:10.738414 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:15.752589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:20.777336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:25.797239 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:30.820115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:35.841672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:40.863790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:45.887589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:50.912643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:30:56.171273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:01.193529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:06.214401 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:11.234161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:16.249900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:21.270130 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:26.404225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:31.423323 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:36.443849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:41.460665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:46.483233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:51.509724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:31:57.025846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:02.040741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:07.059078 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:12.101313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:14.520306 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:32:14.520529 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 13:32:14.520641 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 13:32:14.520733 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 13:32:14.520752 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 13:32:14.520764 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 13:32:14.520787 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 13:32:14.520810 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 13:32:14.520835 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 13:32:14.520845 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 13:32:14.520872 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 13:32:14.520947 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 13:32:14.521019 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 13:32:14.521055 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 13:32:14.521127 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 13:32:14.521163 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 13:32:14.521208 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 13:32:14.521246 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:32:14.521259 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:32:14.521268 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.521273 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.521280 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:32:14.521289 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:32:14.521332 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:32:14.521356 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:32:14.521476 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:32:14.521492 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:32:14.521515 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:32:14.521611 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:32:14.521647 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:32:14.521685 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:32:14.521735 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 13:32:14.521788 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:32:14.521799 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:32:14.521806 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.521811 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.521818 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:32:14.521827 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:32:14.521850 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:32:14.521874 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:32:14.521897 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:32:14.521923 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:32:14.521943 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:32:14.521975 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:32:14.521991 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:32:14.521997 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 13:32:14.522420 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:32:14.522469 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 13:32:14.522480 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 13:32:14.522490 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:32:14.522500 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:32:14.522570 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.522608 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.522636 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:32:14.522661 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:32:14.522693 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:32:14.522731 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:32:14.522785 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:32:14.522820 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:32:14.522842 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:32:14.522910 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:32:14.522948 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:32:14.522977 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:32:14.523020 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 13:32:14.600733 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:32:14.600758 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:32:14.600789 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:32:14.600799 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:32:14.601035 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:32:14.601050 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:32:14.601124 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:32:14.601190 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:32:14.601220 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.601248 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:32:14.601272 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:32:14.601296 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:32:14.601319 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:32:14.601349 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:32:14.601375 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:32:14.601447 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:32:14.601471 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:32:14.601537 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:32:14.601567 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:32:14.601608 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 13:32:14.602227 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 13:32:14.669748 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:32:14.669860 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:32:14.669966 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:32:14.670039 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:32:14.670118 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:32:14.670197 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:32:14.670311 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 13:32:17.118860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:22.140574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:27.224621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:32.249234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:37.269790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:42.292361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:47.310645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:52.351631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:32:57.367814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:02.390513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:07.406034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:12.425216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:17.443877 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:22.464803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:27.483579 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:32.557077 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:37.573722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:42.604110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:47.619811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:52.642420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:33:57.663180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:02.679993 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:06.282832 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 13:34:07.705626 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:12.722258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:17.745960 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:22.765240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:27.786210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:32.810247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:37.829534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:42.847063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:47.869627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:52.887709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:34:57.915046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:02.942722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:07.958115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:12.979910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:17.999980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:21.765114 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 13:35:23.021215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:28.035844 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:33.058197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:38.076115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:43.098392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:48.163081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:53.184318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:35:58.200516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:03.236383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:08.257429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:13.275781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:18.299846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:20.112263 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 13:36:23.346363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:28.371698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:33.397937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:38.419425 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:43.457268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:48.481272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:53.838339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:36:58.867853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:03.898675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:08.922752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:14.036783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:19.066387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:24.088789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:29.113939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:34.137117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:39.155365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:44.175810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:49.198007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:54.219921 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:37:59.242446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:04.262332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:09.287752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:14.327079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:19.353379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:24.371927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:29.405030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:34.423332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:39.451827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:43.225747 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 13:38:44.468017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:49.492517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:54.517920 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:38:59.550625 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:04.568646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:09.600369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:14.618222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:19.637127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:24.669805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:29.692867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:34.737787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:39.757541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:44.784922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:49.808548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:54.878353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:39:59.898038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:04.923717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:09.952082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:14.979519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:19.999148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:25.056050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:30.083642 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:35.102292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:40.118562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:45.145305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:50.161623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:40:55.185332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:00.209907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:05.233934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:10.272756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:15.300584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:20.322677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:25.358817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:30.389813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:35.415954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:40.435157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:45.452119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:50.476521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:41:55.493254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:00.509655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:05.528242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:10.544805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:15.570297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:20.588925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:25.615503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:30.632287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:35.656509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:40.672000 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:45.693213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:50.709740 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:42:55.732864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:00.747723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:05.765935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:10.782580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:15.817538 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:20.837788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:25.862232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:30.886423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:35.904330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:40.924837 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:41.288153 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 8 items received I0617 13:43:45.942898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:47.124177 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 13:43:50.960460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:43:56.342184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:01.367418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:06.388422 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:11.407153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:12.790927 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 13:44:16.427729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:21.456517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:26.796966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:31.836290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:36.861711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:41.890389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:46.909554 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:51.929771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:44:57.158874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:02.179944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:07.198210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:12.215912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:17.238275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:22.259529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:27.279510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:32.301141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:37.317232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:42.335504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:47.361892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:52.380645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:45:57.401766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:02.424892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:07.448560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:12.465703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:17.485437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:22.506800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:27.528909 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:32.548706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:37.570500 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:42.589534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:47.609895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:52.653061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:46:57.673343 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:02.693363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:07.729331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:12.748409 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:14.521482 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:47:14.521561 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 13:47:14.521578 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 13:47:14.521596 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 13:47:14.521610 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 13:47:14.521620 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 13:47:14.521624 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 13:47:14.521632 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 13:47:14.521638 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 13:47:14.521643 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 13:47:14.521658 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 13:47:14.521664 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 13:47:14.521670 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 13:47:14.521674 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 13:47:14.521679 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 13:47:14.521684 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 13:47:14.521687 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 13:47:14.521701 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:47:14.521707 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:47:14.521714 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.521719 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.521726 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:47:14.521734 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:47:14.521739 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:47:14.521745 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:47:14.521752 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:47:14.521756 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:47:14.521761 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:47:14.521819 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:47:14.521831 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:47:14.521835 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 13:47:14.522229 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:47:14.522237 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:47:14.522244 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.522250 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.522255 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:47:14.522262 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:47:14.522268 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:47:14.522277 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:47:14.522283 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:47:14.522288 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:47:14.522307 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:47:14.522343 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:47:14.522351 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:47:14.522355 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:47:14.522361 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 13:47:14.523493 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 13:47:14.523561 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 13:47:14.523570 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 13:47:14.523580 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 13:47:14.523585 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 13:47:14.523592 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.523596 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.523601 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:47:14.523606 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 13:47:14.523610 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 13:47:14.523616 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 13:47:14.523623 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 13:47:14.523629 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:47:14.523634 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:47:14.523667 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 13:47:14.523679 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 13:47:14.523684 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 13:47:14.523689 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 13:47:14.588984 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:47:14.589012 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:47:14.589039 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:47:14.589047 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:47:14.589054 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:47:14.589066 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:47:14.589476 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 13:47:14.589677 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 13:47:14.589842 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 13:47:14.589875 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.589893 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 13:47:14.589914 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 13:47:14.589944 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 13:47:14.589973 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 13:47:14.590007 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 13:47:14.590038 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 13:47:14.590057 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 13:47:14.590083 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 13:47:14.590117 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 13:47:14.590132 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 13:47:14.590140 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 13:47:14.635901 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 13:47:14.635982 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 13:47:14.636009 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 13:47:14.636018 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 13:47:14.636027 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 13:47:14.636040 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 13:47:14.636451 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 13:47:17.772080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:22.792967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:28.051586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:33.076030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:38.094150 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:43.114879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:48.135292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:53.157328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:58.175008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:47:58.243699 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 13:48:03.198123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:08.229703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:13.344519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:18.375962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:23.436448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:28.460536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:33.484211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:38.512872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:43.532867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:48.570874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:53.677730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:48:58.702113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:03.729283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:08.766974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:13.792327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:18.854641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:23.943035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:28.967093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:33.987558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:39.012654 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:44.029797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:49.053765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:54.070251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:49:59.093677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:04.131869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:09.154399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:14.170887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:19.207443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:24.238242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:29.273841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:34.296008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:39.316580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:44.348955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:49.371415 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:54.422768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:50:59.449093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:04.474002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:05.134984 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 13:51:09.495009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:14.518847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:19.545705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:24.822428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:29.851776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:34.873188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:39.895020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:44.927372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:49.294862 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 13:51:49.960934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:51:55.110391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:00.127099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:05.153949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:10.170842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:15.192523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:20.216053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:25.239839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:30.270630 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:35.291488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:40.317198 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:43.800053 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 2 items received I0617 13:52:45.332692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:50.364724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:52:55.382392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:00.404889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:05.422402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:10.447356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:15.505288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:20.529284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:25.570956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:30.591289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:35.639233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:40.684562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:45.700814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:50.732981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:53:55.768248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:00.794156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:05.820049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:10.845857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:15.872091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:20.896915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:25.920588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:30.937005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:35.953635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:40.969736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:45.989245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:51.029787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:54:56.343150 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:01.365826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:06.402718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:11.429764 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:16.449289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:21.479166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:26.497299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:31.517451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:36.533531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:41.555335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:46.573481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:51.605374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:55:56.641344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:01.680460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:06.703636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:11.739799 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:15.142020 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 13:56:16.763546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:21.809155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:27.107048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:32.141872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:37.215456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:42.238667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:47.286901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:52.311023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:56:57.333224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:02.372991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:07.390068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:10.255652 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 13:57:12.411586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:17.431456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:22.449968 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:27.471422 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:32.492855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:37.514559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:42.535335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:47.600686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:52.640423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:57:57.681348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:02.721936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:07.741570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:12.775328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:17.791890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:22.826860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:27.845227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:32.865990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:37.886660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:42.912700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:47.935383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:52.956537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:58:57.977610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:03.004776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:08.062587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:13.085560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:18.110848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:23.142300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:28.167471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:33.196866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:38.228063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:43.261382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:48.285391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:53.359954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 13:59:58.378623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:03.427871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:08.487207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:13.524683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:18.550561 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:20.304219 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 14:00:23.655036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:28.690937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:33.716293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:38.733777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:43.753897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:48.786948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:53.906129 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:00:58.940013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:03.982539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:09.020466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:14.050164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:19.079596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:24.109926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:29.137235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:34.153896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:39.177592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:44.207315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:49.242375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:54.270088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:01:56.864786 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 14:01:59.564632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:04.587827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:09.671261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:14.522298 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:02:14.522557 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 14:02:14.522578 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 14:02:14.522589 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 14:02:14.522618 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 14:02:14.522645 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 14:02:14.522650 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 14:02:14.522658 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 14:02:14.522741 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 14:02:14.522751 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 14:02:14.523215 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 14:02:14.523270 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 14:02:14.523281 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 14:02:14.523285 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 14:02:14.523304 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 14:02:14.523313 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 14:02:14.523328 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 14:02:14.523347 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:02:14.523352 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:02:14.523360 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:02:14.523432 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:02:14.523477 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:02:14.523494 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:02:14.523514 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:02:14.523526 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:02:14.523537 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:02:14.523541 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:02:14.523549 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:02:14.523627 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:02:14.523955 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:02:14.524019 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:02:14.524030 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 14:02:14.524105 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 14:02:14.524117 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 14:02:14.524132 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 14:02:14.524138 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 14:02:14.524256 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 14:02:14.524298 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:02:14.524395 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:02:14.524409 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 14:02:14.524427 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 14:02:14.524439 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 14:02:14.524449 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 14:02:14.524778 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:02:14.524794 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:02:14.524834 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 14:02:14.524864 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 14:02:14.524871 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 14:02:14.524880 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 14:02:14.625953 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:02:14.625973 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:02:14.626190 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:02:14.626723 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 14:02:14.626759 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:02:14.626977 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:02:14.627048 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:02:14.627190 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:02:14.627201 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:02:14.627211 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:02:14.627239 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:02:14.627247 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:02:14.627269 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:02:14.627278 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:02:14.627293 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:02:14.627308 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:02:14.627313 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:02:14.627321 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:02:14.627454 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:02:14.627484 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:02:14.627493 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 14:02:14.718148 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:02:14.718820 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:02:14.719027 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:02:14.719131 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:02:14.719278 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:02:14.719367 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:02:14.719471 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 14:02:14.728023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:19.785378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:22.268121 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 14:02:24.924514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:29.942935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:34.967621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:36.192327 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 14:02:39.988399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:45.007296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:50.045535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:02:55.295649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:00.337932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:05.357841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:10.383296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:15.401226 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:20.421745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:25.456450 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:30.477368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:35.505940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:40.526572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:45.586699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:50.632591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:03:55.669864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:00.687635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:05.710541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:10.728666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:15.756860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:20.773862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:25.791784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:30.807012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:35.828192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:40.844828 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:45.865731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:50.892261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:04:55.985454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:01.089502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:06.122439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:11.146082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:16.164028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:21.184927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:26.200038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:31.228956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:31.308081 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 14:05:36.249265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:41.270417 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:46.285301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:51.309558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:05:56.416625 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:01.438195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:06.471910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:11.490448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:16.513428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:21.529021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:26.613766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:31.628988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:36.651324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:41.670706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:46.697570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:51.714374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:06:56.731913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:01.775422 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:06.791467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:11.824415 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:16.843979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:21.867458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:26.888435 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:31.906436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:36.929436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:41.963486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:46.985214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:52.003198 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:07:52.253316 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 14:07:57.381813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:02.414127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:07.449763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:12.473040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:17.505313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:22.523135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:27.553481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:32.591429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:37.615753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:42.635098 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:47.662707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:52.679530 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:08:57.707669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:02.721156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:07.749876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:12.869438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:17.889874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:22.916320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:27.985975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:33.016308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:38.039451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:43.068209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:48.489030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:53.531249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:09:58.567453 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:03.672459 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:08.696509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:13.720644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:18.749373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:23.779522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:26.276779 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 14:10:28.800935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:33.833093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:38.858256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:43.878401 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:48.900557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:53.923112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:10:58.945609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:03.970388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:08.998389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:14.022918 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:15.879615 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 10 items received I0617 14:11:19.085868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:24.658126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:29.684168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:34.702759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:39.722407 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:44.746708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:49.774108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:54.807975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:11:59.826562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:04.846846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:09.865913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:14.884703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:19.911784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:24.354579 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 14:12:24.941744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:29.979741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:35.004481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:40.057550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:45.089789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:50.112786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:12:55.214554 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:00.241643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:05.264074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:10.286397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:15.303030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:20.330022 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:25.353033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:30.377381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:35.395698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:40.416631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:45.431855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:50.458513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:13:55.517242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:00.553255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:05.592042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:10.629319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:15.651985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:20.675731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:25.918175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:30.944243 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:36.003881 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:41.023569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:46.041942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:51.061848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:14:56.112455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:01.133144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:06.154592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:11.170212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:16.193813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:21.212866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:26.500034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:31.523423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:36.557244 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:41.572233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:46.589737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:51.615491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:15:56.941689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:01.965211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:06.990117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:12.014081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:17.032987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:22.050221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:27.065514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:32.096201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:37.115103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:42.132598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:46.328879 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 14:16:47.145924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:52.172006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:16:57.190155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:02.221703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:07.237614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:12.255300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:14.522843 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:17:14.522951 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 14:17:14.523004 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 14:17:14.523019 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 14:17:14.523032 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 14:17:14.523058 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 14:17:14.523083 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 14:17:14.523124 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 14:17:14.523153 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 14:17:14.523174 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 14:17:14.523201 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 14:17:14.523230 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 14:17:14.523062 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 14:17:14.523298 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 14:17:14.523343 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 14:17:14.523372 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 14:17:14.523390 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 14:17:14.523421 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:17:14.523458 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:17:14.523493 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.523515 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.523568 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:17:14.523600 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:17:14.523646 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:17:14.523698 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:17:14.523516 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 14:17:14.523776 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 14:17:14.523805 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.523822 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.523853 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:17:14.523886 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 14:17:14.523910 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 14:17:14.523933 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 14:17:14.523975 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 14:17:14.523988 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:17:14.523997 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:17:14.524078 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 14:17:14.524106 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 14:17:14.524124 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 14:17:14.524139 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 14:17:14.524166 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:17:14.524221 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 14:17:14.524235 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 14:17:14.524246 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 14:17:14.524255 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 14:17:14.524275 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.524294 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.524302 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:17:14.524314 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 14:17:14.524326 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 14:17:14.524338 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 14:17:14.524364 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 14:17:14.524375 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:17:14.524381 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:17:14.524417 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 14:17:14.524449 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:17:14.524481 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:17:14.524513 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:17:14.524571 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:17:14.524636 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:17:14.524679 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 14:17:14.524596 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 14:17:14.525272 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 14:17:14.525299 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 14:17:14.588479 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:17:14.588502 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:17:14.588795 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:17:14.588817 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:17:14.589065 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:17:14.589137 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:17:14.589297 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:17:14.589309 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:17:14.589317 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.589375 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:17:14.589389 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:17:14.589558 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:17:14.589574 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:17:14.589638 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:17:14.589653 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:17:14.589699 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:17:14.589850 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:17:14.589876 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:17:14.588910 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 14:17:14.589975 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:17:14.589981 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 14:17:14.680001 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:17:14.680025 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:17:14.680046 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:17:14.680056 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:17:14.680388 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:17:14.680412 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:17:14.680679 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 14:17:17.272933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:22.291832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:27.541345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:32.561632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:34.281864 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 14:17:37.577581 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:42.599759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:47.615940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:52.643266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:17:57.659742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:02.686444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:07.701321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:12.726493 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:17.755733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:22.770145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:27.803503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:32.820110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:37.836205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:42.853545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:47.878236 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:52.902212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:18:58.044284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:03.085886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:08.112008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:13.137734 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:18.157357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:23.187957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:28.219335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:33.240826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:35.377901 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 14:19:38.260767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:43.333739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:48.349718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:53.381637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:19:58.421637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:03.491867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:08.528060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:13.548875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:18.569887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:23.686138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:28.711392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:29.912773 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 14:20:33.730613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:38.758901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:43.782401 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:48.818878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:53.928439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:20:58.952657 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:03.977153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:08.996930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:14.028420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:19.056941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:24.113953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:29.133365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:34.150959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:39.167723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:44.191325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:49.209098 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:54.243058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:21:59.261724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:04.281900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:09.307998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:14.328757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:19.348076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:24.422953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:29.438250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:34.455137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:39.470815 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:44.291755 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 14:22:44.492297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:49.512379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:54.536009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:22:59.556883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:04.579904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:09.597738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:14.616858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:19.636461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:24.933716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:29.984991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:35.006914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:40.021201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:45.044327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:50.060042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:23:55.173707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:00.217181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:05.253929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:10.273317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:15.307215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:20.326879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:25.372189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:30.385465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:35.415103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:40.431055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:45.459138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:50.484069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:24:55.590821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:00.613239 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:05.628109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:10.641259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:15.658309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:20.673603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:25.691673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:30.707330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:35.785037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:40.812695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:45.834985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:50.853513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:55.877384 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:25:57.920733 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 2 items received I0617 14:26:00.932855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:06.080555 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:11.113351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:16.140282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:16.336979 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 14:26:21.155855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:26.179166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:31.198299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:36.217524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:41.236425 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:46.257578 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:51.285714 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:26:56.952233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:01.970596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:06.996291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:12.012738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:17.057391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:22.076071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:27.181033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:32.198047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:37.215015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:42.234379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:47.251048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:52.278335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:27:56.316330 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 14:27:57.306211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:02.382916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:07.400755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:12.442774 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:17.458839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:22.474798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:27.490057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:32.509138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:37.541137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:42.562676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:47.582259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:52.640029 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:28:57.655820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:02.687106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:07.706832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:12.736148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:17.758786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:18.407520 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 14:29:22.781395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:27.801006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:32.821542 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:37.845469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:42.864245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:47.883072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:52.937868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:29:58.003657 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:03.028929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:08.049789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:13.073224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:18.094711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:23.113785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:28.129627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:33.180433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:38.196155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:43.214618 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:48.247321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:53.384671 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:30:58.408055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:03.424491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:08.441527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:13.591974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:18.612474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:23.696711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:28.712300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:33.736972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:38.759949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:39.927077 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 14:31:43.786874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:48.803797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:53.897595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:31:58.919377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:03.953951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:08.979980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:14.006331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:14.524032 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:32:14.524122 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 14:32:14.524135 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 14:32:14.524142 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 14:32:14.524154 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 14:32:14.524164 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 14:32:14.524168 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 14:32:14.524175 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 14:32:14.524180 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 14:32:14.524184 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 14:32:14.524195 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 14:32:14.524200 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 14:32:14.524206 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 14:32:14.524209 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 14:32:14.524215 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 14:32:14.524219 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 14:32:14.524223 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 14:32:14.524233 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 14:32:14.524238 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 14:32:14.524245 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.524251 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.524256 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:32:14.524265 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 14:32:14.524269 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 14:32:14.524277 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 14:32:14.524287 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 14:32:14.524291 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:32:14.524298 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:32:14.524348 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 14:32:14.524355 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 14:32:14.524359 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 14:32:14.524365 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 14:32:14.524373 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:32:14.524376 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:32:14.524381 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.524384 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.524387 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:32:14.524392 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:32:14.524397 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:32:14.524403 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:32:14.524412 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:32:14.524416 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:32:14.524419 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:32:14.524428 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:32:14.524438 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:32:14.524443 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 14:32:14.524882 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:32:14.524913 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 14:32:14.524921 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 14:32:14.524928 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 14:32:14.524931 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 14:32:14.524954 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.524963 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.524969 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:32:14.524974 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 14:32:14.524978 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 14:32:14.524986 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 14:32:14.524992 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 14:32:14.524997 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:32:14.525001 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:32:14.525308 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 14:32:14.525318 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 14:32:14.525322 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 14:32:14.525328 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 14:32:14.635641 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:32:14.636901 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:32:14.637496 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:32:14.638075 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:32:14.638203 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:32:14.638301 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:32:14.638412 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:32:14.638614 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:32:14.637699 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 14:32:14.638861 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.638947 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:32:14.639028 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:32:14.639202 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:32:14.639302 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:32:14.639397 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:32:14.639489 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:32:14.639572 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:32:14.639655 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:32:14.639770 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:32:14.639911 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:32:14.640005 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 14:32:14.688041 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:32:14.688064 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:32:14.688116 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:32:14.688127 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:32:14.688134 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:32:14.688148 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:32:14.688315 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 14:32:15.378456 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 14:32:19.030858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:24.058808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:29.073108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:34.105416 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:39.134028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:44.160281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:49.181385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:54.201132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:32:59.240420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:04.266882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:09.287842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:14.319665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:19.341124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:24.364967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:29.387127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:34.408399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:39.436009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:44.453854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:49.487974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:54.541870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:33:59.562401 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:04.587522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:09.602560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:14.622043 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:19.647289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:24.661925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:29.680952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:34.697552 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:39.720420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:44.740477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:49.771943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:54.821286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:34:59.837891 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:04.856818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:09.887984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:14.906677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:19.923920 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:24.969764 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:30.001222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:35.022205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:40.041925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:45.075987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:50.109338 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:35:55.159215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:00.183481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:05.224321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:10.242212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:15.261806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:20.281186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:25.309539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:30.331107 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:35.351335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:40.365055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:40.411466 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 14:36:45.379365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:50.397371 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:36:55.417811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:00.441229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:05.463797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:06.340261 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 14:37:10.488021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:15.504190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:20.517153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:25.822456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:30.846137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:35.866114 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:40.886645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:45.928570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:50.947123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:37:56.143486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:01.163089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:06.252523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:11.270608 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:16.289710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:21.308978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:26.476146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:31.498194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:36.520427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:41.545251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:46.569588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:51.590249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:38:51.935432 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 14:38:56.613381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:01.640366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:06.655261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:11.682690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:16.702419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:21.722356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:26.748835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:31.771497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:36.785647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:41.822122 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:46.835785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:51.860954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:39:57.011558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:02.031775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:07.051561 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:12.075470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:17.103115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:22.122268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:27.286341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:32.303709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:37.326998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:42.343738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:47.368752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:52.391302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:40:57.410376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:02.442591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:07.464121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:12.483207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:17.502048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:22.516947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:27.554112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:32.600966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:37.618303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:42.642613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:47.664204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:52.686438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:41:56.522066 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 10 items received I0617 14:41:57.766242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:02.793013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:07.823761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:12.986004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:18.002145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:23.021979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:28.038730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:30.429122 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 14:42:33.062335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:38.077563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:43.095359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:48.110015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:53.144782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:42:58.163697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:03.181356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:08.203850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:13.222375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:18.238166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:23.302130 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:27.361549 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 14:43:28.319737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:33.335756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:38.355996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:43.402702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:48.468443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:53.734783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:43:58.759957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:03.795676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:08.822350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:13.851229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:18.876271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:24.086668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:29.110221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:34.132744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:39.149324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:44.167492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:49.184351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:54.668045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:44:59.693125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:04.720027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:09.747577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:14.767558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:19.784801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:25.029384 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:30.058144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:35.081705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:40.124421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:45.146255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:50.165471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:45:55.184053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:00.202825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:05.217248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:10.235168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:15.251823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:20.275719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:25.292546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:30.311591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:35.335063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:40.354534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:45.371749 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:50.389702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:46:55.406356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:00.487444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:03.007497 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 14:47:05.515562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:10.527995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:14.524366 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:47:14.524448 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 14:47:14.524464 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 14:47:14.524475 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 14:47:14.524503 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 14:47:14.524519 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 14:47:14.524526 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 14:47:14.524534 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 14:47:14.524541 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 14:47:14.524546 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 14:47:14.524564 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 14:47:14.524575 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 14:47:14.524583 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 14:47:14.524591 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 14:47:14.524597 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 14:47:14.524603 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 14:47:14.524608 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 14:47:14.524622 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:47:14.524631 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:47:14.524640 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:47:14.524652 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:47:14.524660 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:47:14.524669 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:47:14.524676 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:47:14.524685 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:47:14.524697 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:47:14.524709 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:47:14.524720 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:47:14.524778 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:47:14.524794 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:47:14.524800 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 14:47:14.525465 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 14:47:14.525517 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 14:47:14.525530 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 14:47:14.525545 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 14:47:14.525556 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 14:47:14.525563 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 14:47:14.525569 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:47:14.525576 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:47:14.525589 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 14:47:14.525596 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 14:47:14.525605 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 14:47:14.525616 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 14:47:14.525629 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:47:14.525638 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:47:14.525679 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 14:47:14.525691 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 14:47:14.525697 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 14:47:14.525706 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 14:47:14.595380 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:47:14.595402 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:47:14.595429 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:47:14.595438 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:47:14.595444 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:47:14.595455 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:47:14.595513 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 14:47:14.595520 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 14:47:14.595528 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 14:47:14.595533 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 14:47:14.595541 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 14:47:14.595547 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 14:47:14.595553 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 14:47:14.595563 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 14:47:14.595591 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 14:47:14.595597 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 14:47:14.595605 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 14:47:14.595631 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 14:47:14.595647 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 14:47:14.595670 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 14:47:14.596381 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 14:47:14.639201 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 14:47:14.639227 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 14:47:14.639247 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 14:47:14.639258 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 14:47:14.639264 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 14:47:14.639277 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 14:47:14.639727 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 14:47:15.578418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:20.597207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:25.617688 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:26.530410 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 14:47:30.638067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:35.655401 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:40.675581 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:45.698773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:50.714906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:47:55.737985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:00.754511 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:05.772873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:10.789791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:15.808907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:20.895022 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:25.912977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:30.930966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:35.960261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:40.991499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:46.007648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:51.030837 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:48:56.057742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:01.405196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:06.424412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:11.441816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:16.464082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:21.482302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:26.845337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:31.863331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:32.367463 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 14:49:36.883552 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:41.926012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:46.947168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:51.971328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:49:57.391120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:02.434901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:07.461679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:12.485157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:17.508517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:22.522867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:27.576976 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:32.600612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:37.628268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:42.646886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:47.666809 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:52.687707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:50:53.459573 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 14:50:58.144637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:03.214486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:08.255722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:13.282462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:18.303848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:23.336102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:28.355059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:33.446791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:38.473980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:43.494651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:48.519287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:53.624778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:51:58.645433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:03.679524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:08.700072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:13.725596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:18.741732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:23.761345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:28.787947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:33.816790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:38.832525 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:43.856274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:48.875147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:54.565850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:52:59.587932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:04.604231 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:09.623037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:14.643155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:19.661780 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:24.692136 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:29.708081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:33.024047 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 14:53:34.727798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:39.744540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:44.763999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:49.789753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:54.819180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:53:59.845965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:04.864167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:09.888944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:14.904742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:19.929855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:24.971549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:28.538830 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 14:54:30.005769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:35.019823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:40.043447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:45.079363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:50.105513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:54:55.313769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:00.337182 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:05.353297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:10.380241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:15.399190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:20.419690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:25.462131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:30.482952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:35.494222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:40.517757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:45.542562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:50.677719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:55:55.700147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:00.743684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:05.770193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:10.794217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:16.150688 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:21.178014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:26.617963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:31.642317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:36.696643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:41.728981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:46.749299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:51.772974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:56:56.805168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:01.838286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:06.867764 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:11.894402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:16.942306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:21.971523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:27.318342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:32.344290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:37.370411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:42.394807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:47.418790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:52.440039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:57:57.460163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:02.537030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:07.552207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:12.577691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:17.599576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:18.381219 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 14:58:22.633869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:27.649165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:32.668790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:37.687697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:42.711811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:47.780807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:52.804237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:58:57.818634 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:02.845140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:07.859245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:12.884514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:17.901803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:22.920506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:27.943223 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:32.961237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:37.982079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:42.999237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:48.022985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:53.041173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 14:59:58.063172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:03.385737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:08.425769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:13.447023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:18.472989 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:22.474611 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 15:00:23.547658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:26.039931 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 15:00:28.569334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:32.553255 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 15:00:33.593054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:38.632390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:43.649082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:48.678651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:53.729498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:00:58.753518 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:03.768594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:08.791929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:13.837594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:18.860001 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:24.429655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:29.598513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:34.615273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:39.636644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:44.662967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:49.687181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:54.720336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:01:59.743729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:04.779697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:09.811314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:14.525206 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:02:14.525341 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 15:02:14.525377 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 15:02:14.525388 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 15:02:14.525439 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 15:02:14.525441 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 15:02:14.525451 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 15:02:14.525457 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 15:02:14.525462 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 15:02:14.525467 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 15:02:14.525470 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 15:02:14.525472 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 15:02:14.525478 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 15:02:14.525483 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 15:02:14.525488 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 15:02:14.525478 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 15:02:14.525505 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 15:02:14.525517 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:02:14.525548 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:02:14.525554 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:02:14.525562 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.525566 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.525576 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:02:14.525584 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:02:14.525591 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:02:14.525599 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:02:14.525608 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:02:14.525614 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:02:14.525622 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:02:14.525819 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:02:14.525829 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:02:14.525843 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:02:14.525855 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 15:02:14.525860 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:02:14.525523 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:02:14.525871 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.525875 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.525881 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:02:14.525889 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:02:14.525894 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:02:14.525901 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:02:14.525911 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:02:14.525916 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:02:14.525921 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:02:14.525960 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:02:14.525971 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:02:14.525976 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 15:02:14.525894 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 15:02:14.526007 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 15:02:14.526015 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:02:14.526020 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:02:14.526025 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.526029 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.526033 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:02:14.526040 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:02:14.526044 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:02:14.526051 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:02:14.526059 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:02:14.526072 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:02:14.526077 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:02:14.526110 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:02:14.526115 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:02:14.526121 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:02:14.526127 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 15:02:14.617802 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:02:14.617827 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:02:14.617857 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:02:14.618093 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:02:14.618100 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:02:14.618113 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:02:14.618602 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:02:14.618727 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:02:14.618779 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.618847 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:02:14.618930 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:02:14.618972 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:02:14.619034 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:02:14.619076 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:02:14.619130 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:02:14.619142 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:02:14.619151 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:02:14.619175 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:02:14.619243 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:02:14.619250 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 15:02:14.618534 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 15:02:14.662759 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:02:14.662836 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:02:14.662924 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:02:14.662951 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:02:14.662968 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:02:14.663015 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:02:14.665753 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 15:02:14.835803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:19.853511 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:24.949881 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:29.973812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:34.998055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:40.019793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:45.039616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:50.055766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:02:55.082277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:00.098915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:05.178303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:10.193929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:15.223481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:20.237829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:25.258128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:30.278036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:35.297045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:40.317595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:45.336939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:50.358567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:03:55.376801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:00.395570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:05.418123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:10.433342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:15.494221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:20.517988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:26.116257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:31.137700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:36.159637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:41.196489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:46.225728 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:51.246522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:04:56.269049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:01.287884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:06.312952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:11.330586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:16.366752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:21.382925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:26.433947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:30.404877 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 15:05:31.451616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:36.468534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:41.489940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:46.523519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:51.549717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:05:56.567048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:01.590730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:06.609523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:11.635194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:16.702411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:21.725906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:26.813935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:31.833091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:36.849679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:41.876327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:46.901215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:48.561893 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 15:06:51.921223 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:06:57.169867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:02.196048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:07.234910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:12.262084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:17.284348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:22.305844 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:27.326147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:32.344622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:37.372412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:38.487285 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 15:07:42.395804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:47.419328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:48.052821 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 15:07:52.652225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:07:57.673825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:02.727754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:07.747795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:13.103681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:18.128531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:23.161792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:28.218796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:33.243813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:38.269054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:43.283573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:48.319833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:53.358473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:08:58.375717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:03.421943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:08.466428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:13.923757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:19.073187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:24.199392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:29.218099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:34.235106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:39.255609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:44.274278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:49.296314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:54.331563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:09:59.352947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:04.368242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:09.389314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:14.413379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:19.439283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:24.458069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:29.480898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:34.594183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:39.614019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:44.635667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:49.657259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:54.676071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:10:59.694669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:04.711723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:09.731754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:14.748428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:19.772118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:25.143547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:30.161014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:35.181439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:40.203816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:45.222134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:50.251911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:11:55.338477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:00.360936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:05.378847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:10.396373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:15.411555 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:20.427400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:22.414392 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 15:12:25.442866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:30.460977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:35.476306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:40.620982 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:45.678369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:50.716058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:12:55.730579 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:00.746694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:05.770333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:08.057836 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 15:13:10.788233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:15.809831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:20.829623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:25.856572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:30.875895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:35.899667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:40.916882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:46.026047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:51.047406 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:13:56.078653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:01.095977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:06.134059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:11.158639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:16.172073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:21.193646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:25.567668 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 15:14:26.216454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:31.236554 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:36.253652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:40.492474 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 15:14:41.273143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:46.291272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:51.310076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:14:56.359464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:01.377912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:06.396889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:11.419172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:16.431146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:21.450402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:26.467451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:31.497478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:36.515597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:41.529288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:46.547489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:51.576464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:15:56.693146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:01.716005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:06.738322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:11.773475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:16.793088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:21.820832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:27.179316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:32.201069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:37.219799 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:42.241272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:47.256753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:52.281359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:16:57.298614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:02.326212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:07.343052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:12.367819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:14.526212 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:17:14.526415 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 15:17:14.526212 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:17:14.526496 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 15:17:14.526677 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:17:14.526758 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:17:14.526776 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.526839 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.526853 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:17:14.526876 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:17:14.526883 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:17:14.526894 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:17:14.526918 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:17:14.526983 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:17:14.526999 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:17:14.527110 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:17:14.527136 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:17:14.527160 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 15:17:14.527640 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 15:17:14.527763 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 15:17:14.527798 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 15:17:14.527839 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 15:17:14.527881 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 15:17:14.527900 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 15:17:14.527930 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 15:17:14.527972 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 15:17:14.527992 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 15:17:14.528038 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 15:17:14.528068 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 15:17:14.528105 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 15:17:14.528130 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 15:17:14.528148 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 15:17:14.528175 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 15:17:14.528192 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 15:17:14.526510 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:17:14.528247 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:17:14.528267 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.528288 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.528313 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:17:14.528399 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:17:14.528426 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:17:14.528449 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:17:14.528477 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:17:14.528498 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:17:14.528520 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:17:14.528590 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:17:14.528612 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:17:14.528639 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:17:14.528670 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 15:17:14.528703 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:17:14.528728 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:17:14.528750 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.528781 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.528799 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:17:14.528819 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:17:14.528845 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:17:14.528894 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:17:14.528927 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:17:14.528962 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:17:14.528994 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:17:14.529036 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:17:14.529079 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:17:14.529115 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:17:14.529153 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 15:17:14.579953 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:17:14.580252 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:17:14.580334 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:17:14.580850 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:17:14.580866 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:17:14.580878 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:17:14.580974 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:17:14.580981 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:17:14.580989 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.580994 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:17:14.581002 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:17:14.581009 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:17:14.581043 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:17:14.581054 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:17:14.581065 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:17:14.581070 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:17:14.581080 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:17:14.581130 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:17:14.581157 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:17:14.581164 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 15:17:14.580468 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 15:17:14.665924 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:17:14.667322 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:17:14.667605 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:17:14.667704 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:17:14.667810 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:17:14.668000 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:17:14.668544 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 15:17:17.380653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:22.397598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:27.412287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:32.503433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:37.536559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:42.572873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:47.594350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:52.619232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:17:57.639664 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:02.665758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:07.682989 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:12.703544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:17.723071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:19.063669 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 15:18:22.743561 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:27.759719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:32.780353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:37.799229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:42.825159 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:47.838201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:52.993333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:18:58.007628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:03.032965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:08.053792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:13.076641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:18.100694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:23.138481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:28.188320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:33.212809 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:38.238014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:43.272886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:48.303879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:53.397954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:19:58.517082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:03.537288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:08.549765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:13.582049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:18.608469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:24.170200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:26.428789 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 15:20:29.206511 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:34.233356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:39.247950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:44.268770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:49.292789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:54.342083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:20:59.357565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:04.381657 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:09.411063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:14.427479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:19.446053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:24.483794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:29.505009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:34.527538 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:39.539610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:44.565552 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:49.784892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:21:55.027642 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:00.053138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:05.070061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:10.093581 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:15.110693 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:20.126504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:25.175027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:30.188812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:35.208961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:40.223513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:45.236977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:50.262366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:22:54.521584 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 15:22:55.282810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:00.302135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:05.321508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:10.401746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:15.425134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:20.438297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:21.574794 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 15:23:25.486020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:30.499132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:35.514245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:40.536270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:45.555598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:50.573790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:23:55.661152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:00.675348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:05.696010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:10.711129 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:15.727862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:20.743099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:25.759334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:30.839818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:35.862080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:40.886639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:45.937751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:50.953583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:24:55.986704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:01.002986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:06.021750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:11.045070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:16.066139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:21.087695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:26.107315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:31.132160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:36.151013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:37.085992 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 15:25:41.179549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:46.212030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:51.230447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:25:56.379035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:01.394900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:06.418411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:11.434142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:16.458218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:21.476557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:26.696612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:31.719342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:36.735639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:41.755423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:46.777856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:51.808364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:26:56.838609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:01.870992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:06.888888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:11.908828 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:16.925274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:22.049100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:27.424047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:32.442899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:37.458404 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:42.479705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:47.493687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:52.514234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:27:57.534193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:02.559983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:07.580788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:12.602302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:17.621283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:22.640937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:27.663089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:32.681072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:37.705027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:42.719121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:47.435174 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 15:28:47.740306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:52.756934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:28:57.778357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:02.813997 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:07.836237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:12.854225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:17.876699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:22.901666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:27.921442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:32.949278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:37.970895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:42.991317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:48.011792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:53.091572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:29:58.106847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:03.140390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:08.157449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:13.183558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:18.205856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:23.659093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:28.680282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:33.702359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:38.723598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:43.741290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:47.549212 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 15:30:48.760604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:53.812587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:30:58.838710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:03.857286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:08.937550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:13.956159 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:18.977194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:24.012991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:29.035811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:34.053098 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:39.071050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:44.110132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:49.129006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:53.644071 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 15:31:54.150520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:31:59.415578 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:04.434446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:09.451478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:14.473565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:14.527032 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:32:14.527104 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 15:32:14.527120 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 15:32:14.527134 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:32:14.527140 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:32:14.527152 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.527157 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.527165 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:32:14.527172 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:32:14.527178 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:32:14.527189 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:32:14.527199 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:32:14.527205 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:32:14.527213 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:32:14.527270 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:32:14.527280 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:32:14.527286 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:32:14.527295 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 15:32:14.527306 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:32:14.527310 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:32:14.527314 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.527316 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.527320 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:32:14.527325 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:32:14.527329 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:32:14.527334 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:32:14.527339 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:32:14.527342 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:32:14.527346 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:32:14.527367 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:32:14.527379 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:32:14.527383 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 15:32:14.527851 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:32:14.527890 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 15:32:14.527912 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 15:32:14.527918 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 15:32:14.527925 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 15:32:14.527931 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 15:32:14.527936 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 15:32:14.527941 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 15:32:14.527946 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 15:32:14.527950 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 15:32:14.527963 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 15:32:14.527968 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 15:32:14.527975 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 15:32:14.527979 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 15:32:14.527983 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 15:32:14.527987 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 15:32:14.527990 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 15:32:14.527997 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:32:14.528000 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:32:14.528004 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.528007 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.528012 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:32:14.528020 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:32:14.528023 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:32:14.528029 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:32:14.528036 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:32:14.528040 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:32:14.528045 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:32:14.528075 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:32:14.528085 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:32:14.528089 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:32:14.528094 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 15:32:14.586907 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:32:14.587010 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:32:14.587059 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:32:14.587517 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:32:14.587537 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:32:14.587549 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:32:14.587581 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:32:14.587614 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:32:14.587636 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.587654 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:32:14.587692 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:32:14.587718 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:32:14.587744 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:32:14.587788 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:32:14.587818 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:32:14.587879 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:32:14.587381 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 15:32:14.587912 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:32:14.588110 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:32:14.588218 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:32:14.588279 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 15:32:14.635975 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:32:14.636082 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:32:14.636147 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:32:14.636175 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:32:14.636680 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:32:14.636609 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 15:32:14.636771 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:32:19.492948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:24.510347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:29.529267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:34.544790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:39.567455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:44.589272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:49.607481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:54.622149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:32:59.640286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:04.661424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:09.681302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:14.715017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:19.737897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:24.993011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:30.016869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:35.037927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:40.056809 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:45.072443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:50.094357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:33:55.113024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:00.132805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:05.151659 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:10.173287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:15.189562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:20.209599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:25.263040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:30.281232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:35.369422 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:39.098805 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 15:34:40.406037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:45.421358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:50.440286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:34:55.456103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:00.472813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:05.498534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:10.515163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:15.532437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:20.548856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:25.984745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:31.006534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:36.027842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:37.449880 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 15:35:41.043971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:46.073679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:51.093560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:35:56.121802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:01.156990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:06.175291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:11.193323 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:16.229496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:21.246798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:26.269910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:31.286600 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:36.301979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:41.325879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:46.344568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:51.361756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:36:56.380942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:01.398825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:06.415576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:11.435241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:16.449263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:21.471967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:26.489284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:31.511853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:36.529594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:41.548676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:46.564178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:51.581303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:37:56.597508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:01.620847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:06.641041 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:11.659694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:16.684205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:19.649665 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 15:38:21.700004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:26.715146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:31.733032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:36.754339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:41.771083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:46.787097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:51.801565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:38:57.134706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:02.153132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:07.171079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:12.193378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:17.210049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:22.232571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:27.252427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:32.275232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:37.291589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:38.555653 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 10 items received I0617 15:39:42.311982 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:47.327537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:52.349902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:39:57.518497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:02.535961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:07.548475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:12.572152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:17.593776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:22.618373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:27.638961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:32.661912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:37.676545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:42.692358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:47.711206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:48.121271 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 15:40:52.731620 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:40:57.783439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:02.804145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:07.821765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:12.860442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:17.952196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:22.995080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:28.015788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:33.052733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:38.070305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:43.091740 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:48.113306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:53.163489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:41:58.201974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:03.248666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:08.267134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:13.294726 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:18.336700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:23.445939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:28.462744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:33.479607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:38.508632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:43.531277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:48.556082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:53.679885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:42:55.771457 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 15:42:58.705301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:03.724912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:08.740471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:13.758964 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:18.790596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:23.840208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:28.860521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:33.882601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:38.905277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:43.924739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:48.949684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:54.081604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:43:59.104274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:04.121372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:09.138390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:14.158994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:19.274451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:24.546599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:29.712216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:34.734599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:39.755164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:44.775035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:49.793282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:54.826087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:44:59.840147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:04.859461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:09.885297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:14.916275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:19.933104 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:25.096237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:30.114915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:35.133544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:40.150889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:45.168273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:50.185470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:45:55.865895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:00.905367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:05.935162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:10.955653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:15.994018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:21.013386 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:26.039336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:31.063308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:36.084455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:41.103940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:46.123997 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:51.150168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:46:56.169770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:01.201273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:06.217859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:11.236851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:14.528050 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:47:14.528295 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 15:47:14.528360 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 15:47:14.528379 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 15:47:14.528393 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:47:14.528403 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:47:14.528411 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.528420 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.528448 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:47:14.528462 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:47:14.528468 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:47:14.528479 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:47:14.528491 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:47:14.528502 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:47:14.528534 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:47:14.528607 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:47:14.528621 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:47:14.528664 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:47:14.528695 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 15:47:14.528715 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:47:14.528725 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:47:14.528732 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.528737 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.528744 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:47:14.528772 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:47:14.528778 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:47:14.528787 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:47:14.528799 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:47:14.528804 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:47:14.528811 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:47:14.528849 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:47:14.528868 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:47:14.528878 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 15:47:14.528128 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 15:47:14.529392 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 15:47:14.529442 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 15:47:14.529457 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 15:47:14.529472 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 15:47:14.529482 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 15:47:14.529492 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 15:47:14.529517 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 15:47:14.529523 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 15:47:14.529542 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 15:47:14.529587 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 15:47:14.529598 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 15:47:14.529603 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 15:47:14.529609 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 15:47:14.529615 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 15:47:14.529620 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 15:47:14.529653 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 15:47:14.529664 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 15:47:14.529671 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.529678 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.529686 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:47:14.529698 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 15:47:14.529726 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 15:47:14.529740 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 15:47:14.529749 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 15:47:14.529754 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:47:14.529762 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:47:14.529840 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 15:47:14.529847 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 15:47:14.529853 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 15:47:14.529862 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 15:47:14.609688 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:47:14.609714 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:47:14.609735 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:47:14.609743 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:47:14.609749 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:47:14.609760 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:47:14.609785 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 15:47:14.610334 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 15:47:14.610494 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.610284 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 15:47:14.610533 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 15:47:14.610725 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 15:47:14.610754 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 15:47:14.610805 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 15:47:14.610877 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 15:47:14.610956 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 15:47:14.611066 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 15:47:14.611080 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 15:47:14.611200 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 15:47:14.611216 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 15:47:14.611292 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 15:47:14.652394 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 15:47:14.652417 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 15:47:14.652438 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 15:47:14.652446 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 15:47:14.652452 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 15:47:14.653145 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 15:47:14.653091 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 15:47:16.252999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:17.570086 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 8 items received I0617 15:47:21.276174 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:26.368870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:31.394367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:31.667906 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 15:47:36.413722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:41.430005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:46.452514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:51.477856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:47:56.495152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:01.616395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:06.642856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:07.427293 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 15:48:11.661045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:16.682945 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:21.701843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:26.724056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:31.742245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:36.763803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:41.780459 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:46.805583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:51.821412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:48:57.051546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:02.073153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:07.093093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:12.112644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:17.131262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:22.158673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:27.474389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:32.491032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:37.514875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:42.575415 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:47.608289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:52.626083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:49:57.645689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:02.661758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:07.682589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:12.699513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:17.726141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:22.744121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:27.764935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:32.787912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:37.949650 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:42.970096 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:47.996978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:53.016465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:50:58.038339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:03.060212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:08.087599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:13.113241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:18.399706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:23.502702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:28.531535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:33.564672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:38.584462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:43.602973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:48.666122 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:53.769049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:51:58.785514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:03.817515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:08.901777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:13.934042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:18.956892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:24.008985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:29.034007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:32.782990 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 15:52:34.052468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:39.082755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:44.100153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:49.123939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:52:55.046693 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:00.064289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:05.084481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:10.105023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:15.127154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:20.146423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:25.172547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:30.189686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:35.211349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:40.228012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:45.252649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:50.292212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:55.316442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:53:58.580428 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 15:54:00.344188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:05.363138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:10.383781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:15.403490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:20.427060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:25.493993 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:30.522682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:35.435201 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 15:54:35.543725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:40.573976 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:45.592887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:50.620958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:54:55.642642 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:00.663350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:05.682917 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:10.708395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:15.725707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:20.748574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:25.765265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:30.786637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:35.817262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:40.845582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:45.886250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:50.992160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:55:56.013925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:01.032829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:06.055587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:11.077455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:11.673586 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 15:56:16.100222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:21.122521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:26.214864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:31.239966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:36.257156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:41.280862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:46.301972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:51.323347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:56:56.703876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:01.728242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:06.753082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:11.774253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:16.792237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:21.818300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:26.850231 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:31.874444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:36.895140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:42.095004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:47.115601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:52.138487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:57:57.156974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:02.178274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:07.192437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:12.223813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:17.241044 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:22.275115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:24.791012 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 15:58:27.421746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:32.443640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:37.465917 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:42.486412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:47.503840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:52.523712 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:58:57.651559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:02.671427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:07.687257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:12.713007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:17.735387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:19.599838 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 15:59:22.758969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:27.776363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:32.798072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:37.814337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:42.830784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:47.853434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:52.869383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 15:59:57.889766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:02.917297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:07.939177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:12.962748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:17.986796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:23.027714 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:28.049843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:33.069717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:38.103627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:43.126581 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:48.145663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:53.169983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:00:58.189985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:03.211899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:08.234040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:13.366044 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:18.407192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:23.509692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:26.805055 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 16:01:28.528423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:33.548781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:38.581801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:43.606433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:48.631707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:53.722533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:01:58.746596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:03.774009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:08.804350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:13.831739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:14.529045 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:02:14.529324 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 16:02:14.529481 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 16:02:14.529520 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 16:02:14.529564 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 16:02:14.529595 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.529654 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.529679 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:02:14.529730 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 16:02:14.529743 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 16:02:14.529755 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 16:02:14.529767 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 16:02:14.529772 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:02:14.529807 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 16:02:14.529813 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 16:02:14.529821 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.529892 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.529995 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:02:14.529846 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:02:14.530168 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 16:02:14.529045 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:02:14.530151 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 16:02:14.530417 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 16:02:14.530471 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 16:02:14.530520 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 16:02:14.530578 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:02:14.530610 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:02:14.530734 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 16:02:14.530777 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 16:02:14.530841 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 16:02:14.530312 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 16:02:14.531106 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 16:02:14.531139 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 16:02:14.530404 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 16:02:14.531233 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 16:02:14.531288 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 16:02:14.531315 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 16:02:14.531374 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 16:02:14.531395 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 16:02:14.531444 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 16:02:14.531472 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 16:02:14.531498 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 16:02:14.531554 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 16:02:14.531596 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 16:02:14.531685 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 16:02:14.531716 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 16:02:14.531758 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 16:02:14.532132 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 16:02:14.532174 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 16:02:14.532212 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 16:02:14.532232 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 16:02:14.532254 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.532272 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.532306 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:02:14.532326 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 16:02:14.532344 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 16:02:14.532380 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 16:02:14.532407 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 16:02:14.532425 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:02:14.532458 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:02:14.532508 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 16:02:14.532514 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 16:02:14.532520 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 16:02:14.532528 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 16:02:14.588661 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 16:02:14.588953 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 16:02:14.589255 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 16:02:14.589280 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 16:02:14.589355 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 16:02:14.589537 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 16:02:14.589661 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 16:02:14.589672 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 16:02:14.589679 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.589754 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:02:14.589840 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:02:14.589853 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 16:02:14.589932 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 16:02:14.589944 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 16:02:14.589953 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 16:02:14.589972 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:02:14.590106 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 16:02:14.589982 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:02:14.590256 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 16:02:14.590357 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 16:02:14.590368 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 16:02:14.637238 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 16:02:14.637260 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 16:02:14.637761 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 16:02:14.637781 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 16:02:14.637787 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 16:02:14.637894 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 16:02:14.638377 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 16:02:18.852954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:23.969929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:28.991557 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:34.027708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:39.046979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:44.076715 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:48.485978 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 16:02:49.096902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:54.129910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:02:59.154192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:04.179769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:09.208699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:14.240520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:19.265892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:24.385357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:29.404065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:34.423032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:39.514618 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:44.544418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:49.568467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:54.596622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:03:59.613791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:04.640973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:09.660723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:14.703645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:19.733379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:24.765912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:29.781906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:34.809341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:39.825290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:44.845595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:49.860271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:54.886509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:04:59.908012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:04.926269 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:09.976752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:15.090461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:20.111411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:25.163668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:30.192245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:35.211949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:40.233706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:45.250563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:50.272413 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:05:55.287926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:00.317823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:05.339708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:10.366343 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:15.387004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:20.404408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:25.433170 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:30.453449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:35.477355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:40.503256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:41.800077 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 16:06:45.528145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:50.552388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:06:55.605717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:00.645268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:05.663967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:10.679631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:14.616407 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 16:07:15.705731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:20.722718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:25.782012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:30.797306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:35.817342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:40.835905 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:45.854126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:50.873743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:07:55.896688 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:00.919585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:05.943296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:10.967139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:15.986943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:21.001969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:26.281686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:27.813258 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 16:08:31.332307 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:36.347184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:41.380093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:46.396964 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:51.447348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:08:56.474399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:01.503470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:06.533326 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:11.550071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:16.604312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:21.639717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:26.670810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:31.694840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:36.720539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:41.735546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:46.829914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:51.848255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:09:56.885546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:01.903046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:06.926750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:11.953392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:16.971126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:21.989685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:27.008020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:32.038645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:37.059058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:42.077071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:42.503901 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 16:10:47.100681 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:52.130208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:10:57.250147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:02.279517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:07.339007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:12.363248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:17.391173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:22.412184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:27.428825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:32.450806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:37.468810 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:42.487562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:47.508262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:52.535739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:11:57.557147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:02.579088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:07.611281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:12.628954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:17.654091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:22.675371 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:27.736745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:32.756994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:37.776080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:42.792501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:47.814083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:52.831646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:12:57.856309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:02.871585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:07.891758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:12.912419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:17.931695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:22.946081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:27.977172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:33.000489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:38.018305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:43.039468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:48.063776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:53.082619 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:13:58.098504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:03.118711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:08.141100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:13.159988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:18.187482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:23.214383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:28.229378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:28.808382 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 16:14:33.261828 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:38.280771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:43.296961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:48.319462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:53.443627 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:14:58.493746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:03.515998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:08.540802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:08.645267 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 16:15:13.561682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:18.605079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:21.842064 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 16:15:24.076801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:29.102943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:34.122694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:39.146736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:44.165005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:49.198382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:54.244309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:15:59.267506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:04.295944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:09.318961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:14.337612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:19.357982 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:24.377006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:29.478101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:34.496762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:39.523994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:44.542699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:49.575635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:54.604322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:16:59.630224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:04.658816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:09.712937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:14.530399 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:17:14.530462 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:17:14.530497 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 16:17:14.530518 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 16:17:14.530533 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 16:17:14.530553 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 16:17:14.530566 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 16:17:14.530571 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 16:17:14.530583 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 16:17:14.530591 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 16:17:14.530598 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 16:17:14.530613 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 16:17:14.530623 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 16:17:14.530635 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 16:17:14.530640 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 16:17:14.530646 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 16:17:14.530651 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 16:17:14.530656 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 16:17:14.530679 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 16:17:14.530690 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 16:17:14.530701 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 16:17:14.530708 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 16:17:14.530715 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 16:17:14.530725 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:17:14.530733 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:17:14.530745 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 16:17:14.530755 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 16:17:14.530763 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 16:17:14.530777 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 16:17:14.530782 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:17:14.530792 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:17:14.530850 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 16:17:14.530864 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 16:17:14.530871 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 16:17:14.530884 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 16:17:14.530900 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 16:17:14.530909 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 16:17:14.530915 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 16:17:14.530923 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:17:14.530928 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:17:14.530935 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 16:17:14.530940 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 16:17:14.530946 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 16:17:14.530954 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 16:17:14.530959 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:17:14.530965 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:17:14.530986 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 16:17:14.531006 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 16:17:14.531015 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 16:17:14.596441 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 16:17:14.596697 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 16:17:14.596785 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 16:17:14.596844 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 16:17:14.596911 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 16:17:14.596946 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 16:17:14.597336 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 16:17:14.727510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:19.758916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:24.837636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:29.890235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:34.920840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:39.939795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:44.961849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:49.983924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:17:55.005516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:00.024925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:05.043606 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:10.071113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:15.091180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:20.170819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:25.191900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:30.209056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:35.228321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:40.250376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:45.269157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:50.292583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:18:55.314079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:00.332817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:05.349235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:10.373155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:15.393100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:20.411986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:25.427184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:30.447990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:35.464793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:40.513485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:45.536710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:50.555760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:19:55.576433 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 16:19:55.597730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:00.648843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:05.667644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:10.689196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:15.706136 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:20.728685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:25.764732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:30.785688 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:35.801886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:40.829803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:46.045042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:51.067896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:56.083143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:20:59.661687 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 16:21:01.146938 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:06.159953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:11.182061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:16.205314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:21.242716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:26.266537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:31.288461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:36.319391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:41.344256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:46.369133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:51.384704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:21:56.404418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:01.430686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:06.451949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:11.476760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:16.498015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:18.846833 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 16:22:21.660842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:26.681988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:31.701678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:36.720119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:41.737865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:46.757876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:51.790466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:22:56.811181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:01.831063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:06.847802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:11.869147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:14.811804 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 16:23:16.885821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:21.901884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:26.986747 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:32.030800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:37.051250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:42.071792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:47.091212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:52.113878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:23:57.135026 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:02.160658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:07.179005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:12.200088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:17.220311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:22.252862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:27.269206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:32.286451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:37.301737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:42.320645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:47.357016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:52.379875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:24:57.402434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:02.580920 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:07.599505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:12.628954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:17.646601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:22.672310 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:27.694643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:32.713216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:37.727903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:39.583432 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 1 items received I0617 16:25:42.753987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:47.788067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:52.812220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:25:57.834277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:02.850698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:07.869515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:12.887782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:17.904720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:22.934073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:27.951808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:32.972091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:37.992212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:43.010393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:48.041245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:53.058386 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:26:58.078283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:03.094907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:08.111731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:13.135388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:18.155730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:21.668162 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 16:27:23.173999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:28.197046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:33.238171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:38.256540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:43.289977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:48.327219 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:53.410635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:27:58.428426 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:03.461769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:08.498729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:13.513345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:18.536599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:23.597949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:28.617746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:32.833455 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 16:28:33.635825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:38.657527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:43.688641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:48.706951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:53.948844 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:28:58.967768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:03.994211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:09.019046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:14.041269 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:19.147904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:24.183460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:29.201957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:34.217800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:39.233773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:44.252299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:49.265651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:54.326935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:29:59.343731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:04.358080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:09.378121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:14.396382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:19.410218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:24.446319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:29.462966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:34.482937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:39.500542 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:44.516820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:49.536254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:54.573838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:30:59.594875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:04.609453 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:09.637582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:14.651980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:19.671980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:24.688617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:29.718324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:34.735772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:39.754038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:44.773341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:49.797293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:51.859151 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 9 items received I0617 16:31:54.826485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:31:59.852789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:04.871527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:09.886436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:14.531443 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:32:14.531505 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 16:32:14.531515 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 16:32:14.531523 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 16:32:14.531532 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 16:32:14.531540 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 16:32:14.531544 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 16:32:14.531549 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 16:32:14.531555 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 16:32:14.531559 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 16:32:14.531572 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 16:32:14.531576 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 16:32:14.531580 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 16:32:14.531583 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 16:32:14.531586 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 16:32:14.531593 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 16:32:14.531596 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 16:32:14.531603 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:32:14.531626 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 16:32:14.531630 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 16:32:14.531634 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.531638 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.531643 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:32:14.531649 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 16:32:14.531653 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 16:32:14.531657 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 16:32:14.531664 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 16:32:14.531667 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:32:14.531680 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:32:14.531719 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 16:32:14.531729 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 16:32:14.531775 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 16:32:14.531781 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 16:32:14.531790 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 16:32:14.531793 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 16:32:14.531797 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.531800 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.531804 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:32:14.531809 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 16:32:14.531812 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 16:32:14.531816 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 16:32:14.531821 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 16:32:14.531824 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:32:14.531827 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:32:14.531837 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 16:32:14.531847 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 16:32:14.531850 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 16:32:14.532217 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 16:32:14.532226 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 16:32:14.532231 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 16:32:14.532235 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 16:32:14.532240 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.532245 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.532252 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:32:14.532258 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 16:32:14.532261 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 16:32:14.532267 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 16:32:14.532273 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 16:32:14.532278 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:32:14.532283 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:32:14.532306 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 16:32:14.532310 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 16:32:14.532314 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 16:32:14.532320 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 16:32:14.590275 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 16:32:14.590316 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 16:32:14.590628 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 16:32:14.590644 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 16:32:14.590663 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 16:32:14.591110 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 16:32:14.591257 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 16:32:14.591311 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 16:32:14.591358 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 16:32:14.591422 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.591432 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:32:14.591440 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:32:14.591449 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 16:32:14.591504 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 16:32:14.591523 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 16:32:14.591580 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 16:32:14.591590 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:32:14.591600 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:32:14.591623 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 16:32:14.591639 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 16:32:14.591709 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 16:32:14.698877 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 16:32:14.699005 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 16:32:14.699118 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 16:32:14.699172 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 16:32:14.699203 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 16:32:14.699252 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 16:32:14.699889 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 16:32:14.903341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:19.920764 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:24.965598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:29.677948 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 16:32:29.980879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:34.999491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:40.020222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:45.035600 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:50.051884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:32:55.345760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:00.370890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:05.394919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:10.419074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:15.442151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:20.460151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:25.488376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:30.508865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:35.529765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:40.549115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:45.612006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:50.639933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:33:55.720563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:00.738621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:05.753507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:10.767344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:15.785778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:20.801836 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:25.868230 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:30.885793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:35.901224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:40.591234 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 16:34:40.916754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:45.949308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:50.963567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:34:52.843169 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 16:34:56.105907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:01.127327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:06.153140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:11.175225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:16.196018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:21.215369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:26.259769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:31.276158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:36.318535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:41.338038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:46.358402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:51.382152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:35:56.403714 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:01.450438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:06.466454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:11.484843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:16.505170 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:21.628564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:26.658846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:31.676753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:36.694717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:41.712952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:46.755404 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:51.771379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:36:56.904972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:01.929350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:06.948033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:11.999160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:17.016917 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:22.061035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:27.217706 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:32.237479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:37.261939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:42.279423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:47.315832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:52.335862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:37:57.364181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:02.380632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:07.399544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:12.413340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:17.432276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:22.458472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:27.520139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:32.537483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:37.559075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:42.570978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:47.594569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:52.650215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:38:57.672322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:02.702841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:07.730453 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:12.742501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:16.695721 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 16:39:17.759859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:22.777361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:27.794308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:32.825686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:37.842926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:42.858014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:47.875489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:52.897609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:39:57.928985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:02.953652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:07.983478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:13.008978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:15.603670 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 16:40:18.037366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:23.072157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:24.888305 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 16:40:28.089334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:33.105936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:38.124784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:43.147087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:48.167623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:53.201108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:40:58.219424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:03.262516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:08.284016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:13.302119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:18.327939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:23.414156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:28.433999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:33.445897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:38.466065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:43.483208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:48.511732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:41:55.009674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:00.033513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:05.052590 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:10.073653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:15.089205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:20.109958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:25.125532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:30.150020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:35.167108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:40.197309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:45.225336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:50.256016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:42:55.424427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:00.443360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:05.463431 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:10.483890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:14.926417 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 16:43:15.505285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:20.534315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:25.566754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:30.591838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:35.625839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:40.642372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:45.669460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:50.692960 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:43:55.713633 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:00.739951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:05.757486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:10.779936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:15.801091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:20.819008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:25.836853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:30.852603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:35.872332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:40.924291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:45.944464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:50.967037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:44:56.063350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:01.082537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:06.100010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:11.122611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:16.140316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:21.159123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:26.175081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:31.206817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:36.228224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:41.252268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:46.273215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:51.296878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:45:56.338054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:01.356817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:06.377872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:11.397830 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:16.415237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:21.444506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:26.466907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:31.486859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:36.510803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:40.701186 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 16:46:41.530598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:46.548584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:51.572039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:46:56.588551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:01.607769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:06.628412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:11.650413 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:14.532605 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:47:14.532633 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 16:47:14.532673 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 16:47:14.532704 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 16:47:14.532715 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 16:47:14.532727 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 16:47:14.532736 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 16:47:14.532741 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 16:47:14.532747 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 16:47:14.532755 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 16:47:14.532759 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 16:47:14.532780 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 16:47:14.532791 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 16:47:14.532795 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 16:47:14.532799 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 16:47:14.532802 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 16:47:14.532803 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 16:47:14.532809 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 16:47:14.532811 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 16:47:14.532814 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:47:14.532817 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 16:47:14.532821 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 16:47:14.532824 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:47:14.532673 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 16:47:14.532832 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 16:47:14.532835 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 16:47:14.532839 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 16:47:14.532844 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 16:47:14.532848 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 16:47:14.532854 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 16:47:14.532858 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:47:14.532864 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:47:14.532862 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 16:47:14.532871 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 16:47:14.532874 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 16:47:14.532880 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:47:14.532875 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 16:47:14.532954 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 16:47:14.532963 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 16:47:14.532968 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:47:14.532974 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:47:14.532999 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 16:47:14.533007 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 16:47:14.533013 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 16:47:14.533021 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 16:47:14.532889 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:47:14.533035 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 16:47:14.533048 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 16:47:14.533053 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 16:47:14.602766 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 16:47:14.602791 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 16:47:14.602861 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 16:47:14.602952 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 16:47:14.603009 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 16:47:14.603475 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 16:47:14.603279 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 16:47:14.603629 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 16:47:14.603754 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 16:47:14.603771 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 16:47:14.603777 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 16:47:14.603823 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 16:47:14.603837 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 16:47:14.603884 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 16:47:14.603930 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 16:47:14.603944 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 16:47:14.603950 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 16:47:14.603996 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 16:47:14.604022 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 16:47:14.604073 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 16:47:14.604083 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 16:47:14.646527 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 16:47:14.647075 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 16:47:14.647190 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 16:47:14.647273 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 16:47:14.647314 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 16:47:14.647367 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 16:47:14.647465 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 16:47:16.667161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:17.608692 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 16:47:21.689784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:26.709118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:31.729641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:36.743329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:41.762922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:46.802759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:51.820593 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:47:57.156328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:02.174298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:07.193050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:12.214386 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:17.235436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:22.261158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:27.280240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:32.306255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:37.325486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:42.349152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:47.362015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:52.435860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:48:57.459907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:02.489556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:07.510413 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:07.945226 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 5 items received I0617 16:49:12.531452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:17.546002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:22.571729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:27.707221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:32.731772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:37.771166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:42.794418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:47.814611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:52.833480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:49:57.859311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:02.879731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:07.900576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:13.029369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:18.044903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:23.065515 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:28.083066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:33.106104 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:38.138512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:43.159441 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:48.181299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:53.210115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:50:58.228850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:03.249101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:08.267701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:13.288910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:18.307826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:23.509124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:28.528127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:33.548098 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:35.944325 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 16:51:38.563321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:43.591251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:48.625584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:53.665004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:51:58.684240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:03.706210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:08.727855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:13.746205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:18.761389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:23.797843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:28.816981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:33.832521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:38.856142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:43.878536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:48.905988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:54.149949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:52:59.171427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:04.189875 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:09.208374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:14.234422 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:19.248410 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:24.267668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:29.291405 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:34.312561 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:39.328019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:44.348271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:48.614224 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 16:53:49.371770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:54.407692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:53:59.448991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:04.471553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:09.485895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:14.505251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:18.723551 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 16:54:19.521305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:24.541518 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:29.593831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:34.612320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:39.630529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:44.651197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:49.672166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:54.688052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:54:59.706457 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:04.725353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:09.740686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:14.761194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:19.776360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:24.818949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:29.838712 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:34.852845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:39.868432 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:44.882523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:49.903040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:54.921640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:55:59.940376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:04.968028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:09.982288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:14.999996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:20.014008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:25.124971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:30.142070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:35.157757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:40.173705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:45.190560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:50.205878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:56:55.324482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:00.343068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:05.358533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:10.369607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:15.394026 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:20.411946 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:25.441152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:30.463053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:35.497274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:40.511149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:45.545976 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:50.561191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:57:55.577789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:00.605459 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:05.629962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:10.653512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:15.672751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:16.960929 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 16:58:20.696166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:25.746508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:30.761034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:35.777928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:35.951210 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 6 items received I0617 16:58:40.799143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:45.832801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:50.849444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:58:55.869229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:00.888462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:05.907369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:10.925076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:15.939644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:20.957144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:25.994417 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:31.019558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:36.036263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:41.053241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:46.084983 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:51.193725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 16:59:56.341608 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:01.378191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:06.406838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:11.435092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:16.552604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:18.620424 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 17:00:21.572299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:26.601061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:31.636110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:36.659804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:41.689791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:46.713879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:51.739168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:00:56.759902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:01.793926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:06.818198 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:12.155679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:17.191804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:22.212345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:27.811963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:32.832890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:37.854366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:42.868874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:47.890354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:52.904541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:01:57.942957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:02.960855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:07.983391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:13.005201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:14.533480 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:02:14.533560 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 17:02:14.533578 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 17:02:14.533590 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 17:02:14.533609 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 17:02:14.533618 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 17:02:14.533623 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 17:02:14.533630 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 17:02:14.533637 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 17:02:14.533642 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 17:02:14.533656 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 17:02:14.533663 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 17:02:14.533669 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 17:02:14.533673 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 17:02:14.533677 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 17:02:14.533682 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 17:02:14.533686 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 17:02:14.533698 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 17:02:14.533708 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 17:02:14.533716 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.533720 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.533727 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:02:14.533734 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 17:02:14.533740 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 17:02:14.533749 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 17:02:14.533759 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 17:02:14.533764 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:02:14.533773 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:02:14.533816 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 17:02:14.533828 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 17:02:14.533832 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 17:02:14.533838 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 17:02:14.533846 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:02:14.533850 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:02:14.533854 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.533858 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.533863 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:02:14.533870 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:02:14.533956 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:02:14.533963 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:02:14.533969 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:02:14.533972 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:02:14.533977 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:02:14.533992 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:02:14.533924 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:02:14.534033 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 17:02:14.534039 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 17:02:14.534044 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 17:02:14.534048 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 17:02:14.534051 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.534054 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.534059 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:02:14.534066 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 17:02:14.534071 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 17:02:14.534078 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 17:02:14.534083 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 17:02:14.534088 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:02:14.534092 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:02:14.534118 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 17:02:14.534122 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 17:02:14.534126 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 17:02:14.534131 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 17:02:14.534156 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:02:14.534161 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 17:02:14.605477 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:02:14.605586 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:02:14.605693 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:02:14.605768 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:02:14.605777 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:02:14.605848 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:02:14.605930 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:02:14.605942 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:02:14.605950 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.606001 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:02:14.606014 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:02:14.606023 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:02:14.606071 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:02:14.606089 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:02:14.606166 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:02:14.606248 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:02:14.606261 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:02:14.606341 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:02:14.606418 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:02:14.606429 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 17:02:14.607552 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 17:02:14.662487 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:02:14.662727 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:02:14.662880 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:02:14.662953 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:02:14.662978 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:02:14.663073 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:02:14.663515 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 17:02:18.031335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:23.048474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:28.078164 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:33.093068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:38.115842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:43.155568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:48.175952 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:53.282638 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:02:58.299321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:03.368883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:08.390037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:13.422266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:18.446463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:23.493296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:28.511308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:33.539801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:38.568503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:43.587956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:48.607258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:53.709187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:03:58.724763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:03.744058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:08.771351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:11.956337 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 1 items received I0617 17:04:13.800807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:15.730958 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 10 items received I0617 17:04:18.816919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:25.094942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:30.111010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:35.127931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:40.158452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:45.190666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:50.210117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:04:55.259099 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:00.274113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:05.291618 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:10.318730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:15.344864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:20.362411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:25.379060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:30.397149 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:35.412991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:40.437155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:45.460516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:50.476436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:05:55.489294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:00.611473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:05.646174 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:10.661873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:15.680738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:20.697974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:26.072275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:31.091762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:36.107480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:41.125728 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:46.141699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:47.625249 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 17:06:51.157914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:06:56.175339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:01.208111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:06.227994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:11.249804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:16.266278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:21.288975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:26.313688 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:31.329863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:36.356785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:41.375445 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:46.395564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:51.412166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:56.840885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:07:58.970643 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 17:08:01.891710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:06.915501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:11.940896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:16.955687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:21.976878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:27.210743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:32.286094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:37.305193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:42.328179 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:47.347994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:52.368073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:08:57.385564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:02.415483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:07.434186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:12.455490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:17.478075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:22.499220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:27.516742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:32.542639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:37.559738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:42.580894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:47.596291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:52.617500 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:09:57.635950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:02.663176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:02.738422 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 17:10:07.692043 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:12.712107 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:17.728684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:22.743867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:27.761839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:32.783971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:37.803629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:42.820708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:47.843787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:52.860577 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:10:57.880196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:02.898756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:07.918119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:12.936669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:17.954723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:22.975322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:27.998594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:33.019978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:38.040380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:43.063932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:48.084807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:53.117364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:11:58.135797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:03.157654 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:08.176213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:08.644137 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 17:12:13.196012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:18.220778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:23.306137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:28.325227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:33.343984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:38.363539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:43.397334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:48.422786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:53.453510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:12:58.477140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:03.499199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:08.525533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:13.566727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:18.586880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:23.635235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:28.659615 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:33.694477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:38.715349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:38.999814 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 17:13:43.733325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:48.752809 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:53.789224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:13:58.805814 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:03.844564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:08.859850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:08.960987 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 17:14:13.883946 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:18.915766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:24.200918 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:29.398156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:34.454102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:39.477750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:44.497157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:49.515304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:54.541700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:14:59.564443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:04.582876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:09.599274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:14.617801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:19.635271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:24.947395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:30.119666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:35.146190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:40.165290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:45.189647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:50.208552 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:15:55.471173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:00.489969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:05.517888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:10.537348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:15.580108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:20.641820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:26.141073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:31.158463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:36.181118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:41.195703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:46.241364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:51.257611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:16:56.396575 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:01.423416 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:06.452482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:11.475331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:14.533627 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:17:14.533827 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 17:17:14.533903 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 17:17:14.533967 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 17:17:14.534032 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 17:17:14.534092 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 17:17:14.534124 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 17:17:14.534216 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 17:17:14.534261 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 17:17:14.534315 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 17:17:14.534369 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 17:17:14.534443 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 17:17:14.534501 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 17:17:14.534544 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 17:17:14.534677 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 17:17:14.534605 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:17:14.534627 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 17:17:14.534874 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 17:17:14.534914 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.534975 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.535020 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:17:14.535085 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 17:17:14.535122 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 17:17:14.535179 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 17:17:14.535272 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 17:17:14.535341 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:17:14.535388 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:17:14.535583 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 17:17:14.535552 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 17:17:14.535756 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 17:17:14.535853 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 17:17:14.535899 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 17:17:14.535986 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 17:17:14.535564 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 17:17:14.536141 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 17:17:14.536038 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:17:14.536272 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:17:14.536334 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.536397 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.536461 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:17:14.536526 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:17:14.536588 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:17:14.536656 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:17:14.536726 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:17:14.536787 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:17:14.536849 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:17:14.536961 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:17:14.537046 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:17:14.537098 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 17:17:14.536199 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 17:17:14.537514 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 17:17:14.537546 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.537595 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.537634 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:17:14.537682 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 17:17:14.537728 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 17:17:14.537763 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 17:17:14.537813 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 17:17:14.537848 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:17:14.537895 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:17:14.538017 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 17:17:14.538090 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 17:17:14.538186 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 17:17:14.538266 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 17:17:14.608831 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:17:14.608857 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:17:14.608884 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:17:14.608891 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:17:14.608897 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:17:14.608909 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:17:14.608938 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:17:14.608945 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:17:14.608952 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.608958 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:17:14.608966 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:17:14.608973 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:17:14.608978 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:17:14.608988 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:17:14.609000 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:17:14.609005 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:17:14.609012 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:17:14.609032 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:17:14.609047 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:17:14.609052 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 17:17:14.609440 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 17:17:14.667707 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:17:14.667734 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:17:14.667759 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:17:14.667768 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:17:14.667774 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:17:14.667786 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:17:14.668092 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 17:17:16.500562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:21.531007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:26.551154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:31.571520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:36.599907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:41.621824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:46.649725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:51.671391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:17:53.793583 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 17:17:56.941061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:02.011752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:07.033227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:12.055147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:17.081821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:22.103752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:27.138514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:32.156279 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:37.191866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:42.219349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:47.241271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:52.257393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:18:57.338333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:02.497212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:07.518267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:12.542089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:17.569501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:22.776419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:27.798356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:32.828813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:37.848793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:42.868767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:47.891266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:52.912932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:19:57.930584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:02.950735 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:05.657871 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 17:20:08.000181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:13.021046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:18.044673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:23.073783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:28.093030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:33.114090 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:38.136333 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:43.309481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:48.326995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:53.364265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:20:58.428335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:03.453752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:08.477543 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:13.499286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:18.523471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:23.725661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:28.753592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:33.771408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:38.836175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:43.852194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:48.887304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:54.010992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:21:59.044746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:04.061765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:09.081292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:10.972936 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 17:22:14.097776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:19.129573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:24.198231 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:29.225321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:34.251299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:39.269905 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:44.295509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:49.324531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:54.371288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:22:59.396161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:04.416446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:09.454344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:13.006287 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 9 items received I0617 17:23:14.474410 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:19.496874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:24.515651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:29.533263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:34.564900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:39.586262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:44.627602 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:49.651196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:54.779634 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:23:59.798896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:04.820447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:09.839962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:14.862271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:19.879988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:24.899259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:29.916592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:33.822386 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 17:24:34.940722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:39.957662 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:44.974509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:49.994817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:24:55.302455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:00.320938 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:05.337448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:10.353312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:15.368295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:20.382501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:25.401902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:30.421650 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:35.438030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:40.466034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:45.487436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:50.506833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:25:55.572424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:00.594581 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:05.615528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:10.636766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:15.654504 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:20.677297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:25.691192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:30.714228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:35.730036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:40.753746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:45.771474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:50.791845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:26:56.127722 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:01.149794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:06.177239 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:11.194533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:16.217611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:21.237106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:26.264552 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:31.284241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:36.462167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:41.485748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:43.673734 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 17:27:46.507783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:51.523911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:27:56.558420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:01.578858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:06.602292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:11.619870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:13.985915 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 17:28:16.641017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:21.659968 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:27.128971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:32.147470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:37.168757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:42.185468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:47.207342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:52.225606 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:28:57.247608 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:02.387958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:07.410449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:12.432967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:17.454912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:22.487705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:27.504930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:32.523498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:37.546822 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:42.569186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:47.587801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:52.619476 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:29:57.637605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:02.668770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:07.687967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:12.712900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:17.729079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:22.799158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:27.815737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:32.840955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:37.854977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:42.875321 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:47.891761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:52.915775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:30:57.930710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:02.964839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:07.995685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:13.015617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:18.046934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:23.071367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:28.099396 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:33.136087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:38.170218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:43.208007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:48.234562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:53.292204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:31:58.311521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:03.338192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:08.357252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:13.382114 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:14.534608 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:32:14.534815 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 17:32:14.534936 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 17:32:14.535019 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 17:32:14.535115 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 17:32:14.535202 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 17:32:14.535216 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 17:32:14.535226 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 17:32:14.535234 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 17:32:14.535240 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 17:32:14.534911 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:32:14.535280 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 17:32:14.535304 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 17:32:14.535314 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.535323 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.535331 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:32:14.535341 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 17:32:14.535352 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 17:32:14.535487 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 17:32:14.535500 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 17:32:14.535511 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:32:14.535597 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 17:32:14.535611 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 17:32:14.535522 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:32:14.535695 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 17:32:14.535621 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:32:14.535758 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:32:14.535765 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.535771 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.535779 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:32:14.535794 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:32:14.535799 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:32:14.535809 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:32:14.535820 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:32:14.535893 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:32:14.535866 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 17:32:14.536003 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 17:32:14.535950 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:32:14.536145 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:32:14.536199 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:32:14.536240 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 17:32:14.536077 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 17:32:14.536453 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 17:32:14.536463 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 17:32:14.536544 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.536554 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.536629 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:32:14.536642 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 17:32:14.536748 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 17:32:14.536820 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 17:32:14.536834 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 17:32:14.536838 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:32:14.536845 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:32:14.536863 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 17:32:14.536869 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 17:32:14.534999 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 17:32:14.537052 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 17:32:14.537085 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 17:32:14.537024 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 17:32:14.537166 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 17:32:14.537196 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 17:32:14.537205 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 17:32:14.537213 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 17:32:14.537218 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. E0617 17:32:14.591540 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:32:14.591565 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:32:14.591845 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:32:14.591865 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:32:14.591872 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:32:14.592307 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:32:14.591991 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 17:32:14.592533 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:32:14.592545 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:32:14.592589 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.592596 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:32:14.592604 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:32:14.592612 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:32:14.592619 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:32:14.592691 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:32:14.592723 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:32:14.592763 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:32:14.592841 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:32:14.592927 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:32:14.592988 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:32:14.593099 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 17:32:14.633274 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:32:14.633530 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:32:14.633615 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:32:14.633668 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:32:14.633730 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:32:14.633764 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:32:14.633921 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 17:32:18.418250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:23.476131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:28.502835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:33.538397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:38.553866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:43.586662 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:48.610561 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:54.365853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:32:59.495800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:04.518367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:06.020009 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 17:33:09.533533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:14.552472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:19.570948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:24.607672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:29.639650 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:34.663086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:39.682304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:44.710761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:49.745031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:54.776853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:33:59.798304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:04.828210 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:06.826737 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 9 items received I0617 17:34:09.848883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:10.996013 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 17:34:14.865136 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:19.886158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:24.911059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:29.936274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:34.968570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:39.990533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:45.025199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:49.683411 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 17:34:50.042211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:34:55.068460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:00.111906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:05.136438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:10.153935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:15.184600 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:20.204013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:25.572962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:30.591669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:35.617260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:40.635702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:45.670768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:50.695358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:35:55.722052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:00.736826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:05.761274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:10.786085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:15.809455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:20.824282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:25.859054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:31.013559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:36.040093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:41.054812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:46.073365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:51.087838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:36:56.109302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:01.127552 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:06.146215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:11.167472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:16.189317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:21.212080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:26.237341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:31.262985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:36.281553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:41.334607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:46.370974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:51.387314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:37:56.405756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:01.425191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:06.440466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:11.459227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:16.480904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:21.506339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:26.555350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:31.575341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:36.602740 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:41.619659 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:46.645665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:51.663187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:38:56.685214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:01.703166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:06.725340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:08.114526 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 17:39:11.739622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:16.758336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:21.771124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:26.837854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:31.851631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:36.872009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:41.894008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:46.917955 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:51.933382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:57.129695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:39:57.834336 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 17:40:02.190444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:07.209123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:12.227643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:17.249588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:22.269325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:27.635175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:32.658249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:37.683537 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:42.703398 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:47.730610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:52.747972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:40:57.764794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:02.787491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:07.809736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:12.827111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:17.846974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:22.863785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:27.882935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:32.902583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:37.926941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:42.958995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:48.001175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:48.702588 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 17:41:53.024319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:41:58.050022 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:03.074957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:08.096998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:13.123465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:18.144949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:23.182917 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:28.201523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:33.225584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:38.245904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:43.300147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:48.321979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:53.384849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:42:58.401923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:03.437638 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:08.469161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:13.490255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:18.513241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:23.618823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:28.664818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:33.690327 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:37.005954 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 10 items received I0617 17:43:38.712120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:43.729385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:48.783371 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:53.850644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:43:58.883600 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:03.916761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:08.943471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:13.962805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:19.017211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:24.050346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:29.071986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:34.089569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:39.117658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:44.134605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:49.159147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:54.201665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:44:59.301862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:04.321204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:09.343742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:14.361533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:19.384808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:24.397641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:29.420140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:34.446467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:39.470972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:44.494566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:49.516928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:54.543079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:45:59.563694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:04.587117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:09.607842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:14.630886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:19.653013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:24.677379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:29.703026 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:34.723887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:39.739863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:44.760017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:49.779177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:54.817300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:46:59.840829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:04.858580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:09.894045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:14.535637 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:47:14.535707 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 17:47:14.535729 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 17:47:14.535738 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 17:47:14.535746 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 17:47:14.535755 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 17:47:14.535762 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 17:47:14.535776 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 17:47:14.535783 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 17:47:14.535787 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 17:47:14.535791 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 17:47:14.535802 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 17:47:14.535806 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 17:47:14.535811 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 17:47:14.535814 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 17:47:14.535817 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 17:47:14.535822 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 17:47:14.535825 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 17:47:14.535834 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:47:14.535837 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:47:14.535853 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:47:14.535857 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:47:14.535862 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:47:14.535868 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:47:14.535871 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:47:14.535876 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:47:14.535883 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:47:14.535886 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:47:14.535943 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 17:47:14.535952 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 17:47:14.535958 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 17:47:14.535961 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 17:47:14.535964 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 17:47:14.535967 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:47:14.535971 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:47:14.535975 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 17:47:14.535978 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 17:47:14.535983 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 17:47:14.535988 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 17:47:14.535992 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:47:14.535899 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:47:14.536029 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:47:14.536039 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:47:14.536043 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 17:47:14.536082 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:47:14.536097 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 17:47:14.536103 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 17:47:14.536107 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 17:47:14.536113 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 17:47:14.587413 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:47:14.587505 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:47:14.587601 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:47:14.588084 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:47:14.588370 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:47:14.588516 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:47:14.587886 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 17:47:14.588606 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 17:47:14.588687 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 17:47:14.588777 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 17:47:14.588814 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 17:47:14.589013 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 17:47:14.589076 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 17:47:14.589112 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 17:47:14.589171 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 17:47:14.589203 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 17:47:14.589218 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 17:47:14.589267 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 17:47:14.589309 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 17:47:14.589342 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 17:47:14.589349 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 17:47:14.642246 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 17:47:14.642271 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 17:47:14.642297 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 17:47:14.642307 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 17:47:14.642314 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 17:47:14.642326 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 17:47:14.642794 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 17:47:14.907431 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:19.926975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:24.980205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:30.003250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:35.020100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:40.039860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:45.061039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:50.079481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:47:55.285548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:00.305991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:05.322939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:10.359793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:15.377191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:20.396398 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:25.423996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:30.445736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:35.463717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:40.489080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:45.502614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:50.522586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:55.577762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:48:57.841210 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 17:49:00.121276 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 17:49:00.640227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:05.660012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:10.684360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:15.904479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:20.939290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:26.065087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:31.082701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:33.754794 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 9 items received I0617 17:49:36.096328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:41.131855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:46.143837 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:51.167192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:49:56.187256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:01.205783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:06.231869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:11.249828 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:16.301772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:21.319191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:26.557220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:31.583944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:32.024734 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 6 items received I0617 17:50:36.604138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:41.627141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:46.662849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:51.725751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:50:57.108008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:02.131258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:07.151989 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:12.174463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:17.199601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:22.227562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:27.457737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:32.477736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:37.500676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:42.521525 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:47.546892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:52.569285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:51:57.697265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:02.720002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:07.741344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:12.758961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:17.781786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:22.803576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:27.849029 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:32.868045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:37.891382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:42.911901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:47.931251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:52.950350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:52:57.990951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:03.032463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:08.053125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:13.078981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:18.097746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:23.114613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:28.135902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:33.160874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:38.189235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:43.215749 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:48.237818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:53.388156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:53:58.406407 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:03.435368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:08.452637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:13.478204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:18.496709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:23.605276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:28.632573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:33.666724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:38.703580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:43.729812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:48.759622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:53.801996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:54:58.822329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:03.847865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:08.902994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:13.930075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:18.954739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:24.025028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:29.056001 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:34.080254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:39.100896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:44.136241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:49.158729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:54.184217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:59.202382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:55:59.761914 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 17:56:04.224002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:09.243141 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:14.265836 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:19.284188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:24.752784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:29.873700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:34.905947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:39.924104 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:44.949913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:49.973236 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:56:55.030906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:00.049528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:05.072225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:05.215498 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 17:57:10.085841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:15.105470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:20.122640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:25.177646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:30.193234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:35.215807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:40.230085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:45.247509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:50.263519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:57:55.284888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:00.301920 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:05.375848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:10.394249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:15.416394 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:20.440247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:25.470606 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:30.491611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:35.510456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:39.034064 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 17:58:40.532402 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:45.553751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:50.575793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:58:50.870573 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 17:58:55.601950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:00.620018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:05.647378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:10.665345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:15.690032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:20.706726 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:26.030424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:31.057287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:36.077848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:41.096795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:46.120818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:51.139701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 17:59:56.163007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:01.179838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:06.197727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:11.213684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:16.247360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:21.265250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:26.286727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:31.302408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:36.330117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:41.342502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:46.364927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:51.383529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:00:56.401262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:01.421909 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:06.444038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:11.464268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:16.483158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:21.501288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:26.521761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:31.545181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:36.563691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:41.591711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:46.607358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:51.629069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:01:56.647032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:01.672294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:06.712793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:11.730254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:14.536741 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:02:14.536979 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 18:02:14.537031 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 18:02:14.537060 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:02:14.537080 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:02:14.537103 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.537121 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.537141 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:02:14.537164 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:02:14.537182 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:02:14.537205 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:02:14.537241 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:02:14.537261 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:02:14.537281 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:02:14.537343 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:02:14.537364 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:02:14.537381 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:02:14.537430 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 18:02:14.537464 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 18:02:14.537482 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 18:02:14.537499 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.537514 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.537533 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:02:14.537551 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 18:02:14.537568 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 18:02:14.537588 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 18:02:14.537610 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 18:02:14.537627 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:02:14.537644 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:02:14.537679 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 18:02:14.537705 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 18:02:14.537722 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 18:02:14.538260 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:02:14.538377 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 18:02:14.538395 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 18:02:14.538406 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 18:02:14.538416 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 18:02:14.538426 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 18:02:14.538434 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 18:02:14.538440 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 18:02:14.538447 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 18:02:14.538458 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 18:02:14.538477 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 18:02:14.538488 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 18:02:14.538496 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 18:02:14.538500 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 18:02:14.538505 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 18:02:14.538510 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 18:02:14.538514 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 18:02:14.538529 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:02:14.538539 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:02:14.538546 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.538555 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.538562 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:02:14.538577 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:02:14.538590 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:02:14.538597 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:02:14.538606 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:02:14.538612 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:02:14.538621 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:02:14.538658 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:02:14.538674 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:02:14.538681 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:02:14.538688 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 18:02:14.611309 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 18:02:14.611333 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 18:02:14.611354 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 18:02:14.611361 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 18:02:14.611367 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 18:02:14.611380 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 18:02:14.611403 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 18:02:14.611870 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 18:02:14.611924 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.611946 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:02:14.611966 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:02:14.611986 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 18:02:14.612021 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 18:02:14.612049 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 18:02:14.612085 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 18:02:14.612111 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:02:14.611453 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 18:02:14.612142 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:02:14.612357 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 18:02:14.612425 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 18:02:14.612436 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 18:02:14.664981 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 18:02:14.665005 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 18:02:14.665055 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 18:02:14.665064 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 18:02:14.665070 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 18:02:14.665083 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 18:02:14.665555 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 18:02:16.751255 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:21.769008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:26.832930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:31.849616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:36.872346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:41.978886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:47.000478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:52.019324 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:02:57.044079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:02.073848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:07.089649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:12.116115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:17.140188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:22.171457 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:27.501956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:32.520488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:37.541256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:42.560530 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:47.579998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:52.610342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:03:57.768996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:02.794856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:07.824967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:09.238443 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 18:04:12.845016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:17.862247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:22.884376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:27.904313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:32.929506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:37.958950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:42.982316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:48.016178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:53.036763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:04:58.053738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:03.071116 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:08.096181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:13.340330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:18.372853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:23.681950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:28.699194 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:33.842981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:38.864232 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:43.910094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:48.941829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:52.041286 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 18:05:53.971683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:05:56.769470 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 18:05:58.992811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:04.023700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:09.040478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:14.061376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:19.076319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:24.184498 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:29.212535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:34.233847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:39.259052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:44.276944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:49.307204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:54.362423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:06:59.384664 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:04.401707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:09.424176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:14.459282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:19.488279 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:25.191084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:30.207397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:35.225123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:40.246119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:43.882952 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 18:07:45.266097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:50.284806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:07:55.312870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:00.330599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:05.346411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:10.365134 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:15.394488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:20.414389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:25.443162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:30.464203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:35.478701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:40.499013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:45.512826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:50.545610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:08:55.599967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:00.618003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:05.631411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:10.648583 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:15.676786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:20.697888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:25.718741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:30.738847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:35.760705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:40.786163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:42.260919 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 18:09:45.803874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:50.825323 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:09:55.850229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:00.875594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:05.939289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:10.957969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:15.980176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:21.002049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:26.017933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:31.040281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:36.065564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:41.085612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:46.104478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:51.125156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:10:56.144815 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:01.171125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:06.200783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:11.222775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:16.252005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:21.271861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:26.315088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:31.333496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:36.355392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:41.370979 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:46.392124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:51.415707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:11:56.600075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:01.627440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:06.646391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:11.671130 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:16.690021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:21.712251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:25.789579 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 18:12:26.989750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:32.050416 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:37.068748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:42.092533 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:47.111756 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:52.137951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:12:57.153623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:02.178404 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:07.191389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:12.209526 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:17.226487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:22.245956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:27.269531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:32.289984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:37.316689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:42.334336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:47.351115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:52.367061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:13:57.387502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:02.436393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:07.553491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:07.899201 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 18:14:12.572595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:17.593999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:22.615275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:27.045570 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 9 items received I0617 18:14:27.649930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:32.697580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:37.717287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:42.740277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:47.761273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:52.784236 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:14:57.802748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:02.820858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:07.839396 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:12.877819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:17.893876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:22.918383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:27.933015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:32.267236 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 18:15:32.967876 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:37.991051 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:43.015587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:48.038348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:53.059465 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:15:58.079833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:03.102773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:08.124977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:13.154023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:18.179111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:23.199652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:28.227497 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:33.254758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:38.282599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:43.335716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:48.378013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:53.485885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:16:58.508900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:03.534785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:08.553005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:13.575541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:14.537778 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:17:14.538091 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 18:17:14.538156 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 18:17:14.538195 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:17:14.538243 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:17:14.538275 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.538319 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.538353 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:17:14.538404 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:17:14.538415 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:17:14.538427 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:17:14.538440 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:17:14.538555 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:17:14.538479 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 18:17:14.538649 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 18:17:14.538658 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.538663 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.538670 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:17:14.538679 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 18:17:14.538689 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 18:17:14.538707 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 18:17:14.538721 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 18:17:14.538732 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:17:14.538607 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:17:14.538762 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:17:14.538813 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:17:14.538836 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:17:14.538902 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 18:17:14.538501 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:17:14.538740 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:17:14.539082 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 18:17:14.539152 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 18:17:14.539206 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 18:17:14.539696 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 18:17:14.539775 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 18:17:14.539809 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 18:17:14.539860 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 18:17:14.539907 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 18:17:14.539961 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 18:17:14.539999 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 18:17:14.540045 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 18:17:14.540080 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 18:17:14.540133 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 18:17:14.540164 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 18:17:14.540219 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 18:17:14.540261 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 18:17:14.540321 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 18:17:14.540361 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 18:17:14.540454 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 18:17:14.540522 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:17:14.540546 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:17:14.540590 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.540632 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.540673 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:17:14.540721 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:17:14.540736 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:17:14.540747 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:17:14.540758 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:17:14.540791 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:17:14.540815 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:17:14.540911 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:17:14.540927 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:17:14.540939 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:17:14.540980 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 18:17:14.627033 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 18:17:14.627061 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 18:17:14.627122 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 18:17:14.627136 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 18:17:14.627144 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 18:17:14.627641 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 18:17:14.627157 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 18:17:14.627840 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 18:17:14.627852 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 18:17:14.627899 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.627939 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:17:14.627997 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:17:14.628011 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 18:17:14.628018 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 18:17:14.628061 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 18:17:14.628101 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 18:17:14.628169 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:17:14.628238 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:17:14.628314 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 18:17:14.628422 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 18:17:14.628434 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 18:17:14.679035 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 18:17:14.679064 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 18:17:14.679354 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 18:17:14.679370 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 18:17:14.679390 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 18:17:14.679404 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 18:17:14.679855 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 18:17:18.608056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:23.679120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:28.695961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:33.719572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:38.754439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:41.795714 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 18:17:43.777366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:48.797753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:54.117719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:17:59.512740 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:04.537424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:09.559003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:14.590473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:19.612023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:24.643773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:29.663406 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:34.693385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:39.710913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:44.737686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:49.757087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:54.791144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:18:59.809428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:04.830560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:09.847289 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:14.874963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:19.890929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:24.999163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:30.021487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:35.048570 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:40.063596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:45.086565 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:50.186713 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:19:55.207649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:00.227631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:05.244025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:10.263443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:13.052597 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 18:20:15.282935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:20.304874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:25.326984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:30.343945 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:35.382922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:40.400884 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:45.425651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:50.443673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:20:55.467798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:00.482303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:05.501585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:10.516549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:15.546351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:20.563766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:25.601696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:30.627444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:35.647582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:40.666132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:45.683479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:50.699666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:21:55.805510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:00.837377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:05.868201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:11.085763 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:13.908983 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 18:22:16.103163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:21.130448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:26.168819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:31.186396 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:36.207890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:41.222989 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:45.284894 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 9 items received I0617 18:22:46.245225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:51.258882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:22:56.285385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:01.305776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:06.331015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:11.356156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:16.374092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:21.394040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:26.430071 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:31.452083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:36.471257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:41.488325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:46.516097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:51.533357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:23:56.557738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:01.587958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:06.602788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:11.621705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:16.639376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:21.665990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:26.938198 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:31.968563 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:37.005776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:42.047765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:47.078039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:52.100044 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:24:57.243306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:02.262973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:07.285922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:12.306745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:17.338211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:22.408420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:27.437292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:32.455005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:37.481755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:42.501769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:47.531034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:52.551047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:57.571842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:25:58.061965 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 18:26:02.590682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:07.701605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:12.715551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:17.740571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:22.762464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:27.841443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:32.858797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:37.882452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:42.897698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:43.915299 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 18:26:47.921167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:52.938361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:26:57.961655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:02.986864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:08.007175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:13.026930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:18.047741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:23.068559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:28.087216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:33.103551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:38.130229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:43.155010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:48.183302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:53.262678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:27:58.286019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:03.445874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:08.497604 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:13.520923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:18.590490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:23.646458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:28.678734 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:33.699827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:38.735437 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:43.771599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:48.794265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:53.867923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:28:58.891082 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:03.932896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:08.303417 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 18:29:08.953539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:13.980568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:19.007921 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:24.036580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:29.061102 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:34.085916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:39.113485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:44.138592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:49.159657 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:54.201253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:29:59.223004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:04.244907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:09.262895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:14.298165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:19.317489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:24.905696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:27.061602 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 18:30:29.937750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:34.960127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:39.986587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:45.009893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:50.027505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:30:55.054911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:00.076165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:05.095832 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:10.111686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:15.139379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:20.155214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:25.233685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:30.257464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:35.275910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:40.298247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:45.323573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:50.354187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:31:55.669792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:00.692730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:05.710062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:10.730390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:14.538650 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:32:14.538918 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 18:32:14.539038 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 18:32:14.539131 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:32:14.539204 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:32:14.539279 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.539350 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.539397 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:32:14.539428 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:32:14.539448 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:32:14.539474 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:32:14.539509 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:32:14.539539 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:32:14.539625 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:32:14.539562 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:32:14.539839 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:32:14.539807 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 18:32:14.539895 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 18:32:14.539908 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.539921 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.539930 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:32:14.539942 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 18:32:14.539949 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 18:32:14.539959 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 18:32:14.539973 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 18:32:14.539982 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:32:14.539989 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:32:14.540035 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 18:32:14.540050 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 18:32:14.540056 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 18:32:14.539817 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 18:32:14.540099 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 18:32:14.540121 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 18:32:14.540154 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 18:32:14.540184 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 18:32:14.540203 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 18:32:14.540223 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 18:32:14.540253 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 18:32:14.540279 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 18:32:14.540313 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 18:32:14.540386 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 18:32:14.540423 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 18:32:14.540443 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 18:32:14.540460 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 18:32:14.540475 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 18:32:14.540503 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 18:32:14.539879 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:32:14.540556 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:32:14.540578 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 18:32:14.540625 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:32:14.540680 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:32:14.540728 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.540756 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.540779 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:32:14.540799 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:32:14.540832 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:32:14.540864 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:32:14.540889 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:32:14.540907 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:32:14.540939 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:32:14.541007 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:32:14.541032 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:32:14.541093 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:32:14.541132 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 18:32:14.646728 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 18:32:14.647018 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 18:32:14.647194 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 18:32:14.647374 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 18:32:14.647465 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 18:32:14.647993 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 18:32:14.648185 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 18:32:14.648328 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 18:32:14.648451 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.648537 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:32:14.648710 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:32:14.648814 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 18:32:14.648900 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 18:32:14.649001 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 18:32:14.649094 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 18:32:14.649182 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:32:14.649257 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:32:14.649377 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 18:32:14.649493 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 18:32:14.649575 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 18:32:14.647338 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 18:32:14.690836 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 18:32:14.691099 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 18:32:14.691297 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 18:32:14.692136 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 18:32:14.692287 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 18:32:14.692374 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 18:32:14.691758 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 18:32:15.751721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:20.771655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:25.816322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:30.844092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:35.922858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:40.941914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:45.978468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:50.995640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:32:56.018291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:01.040139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:06.064199 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:11.083283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:16.112943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:21.130667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:26.155586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:31.176368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:36.194399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:41.216513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:46.243760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:47.067687 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 18:33:51.260999 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:33:56.606560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:01.624168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:06.651675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:07.928674 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 18:34:11.669271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:16.689317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:21.707558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:26.725529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:31.746121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:36.766668 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:41.796068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:46.811858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:51.832704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:34:57.090775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:02.113200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:07.131272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:12.156566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:17.174473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:22.199670 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:27.213320 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:28.310279 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 18:35:32.233964 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:37.250196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:42.272846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:47.294584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:52.320455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:35:57.337226 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:02.361216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:07.388003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:12.419294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:17.449827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:20.081632 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 18:36:22.471400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:27.489261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:32.508304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:37.525710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:42.542551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:47.565934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:52.583705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:36:57.760956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:02.782770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:07.801892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:12.829733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:17.854745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:22.888115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:27.904945 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:32.931140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:37.955249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:42.976948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:48.001393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:53.019847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:37:58.035100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:03.055392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:08.070506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:13.097514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:18.110761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:23.143598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:28.163027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:33.188188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:38.206011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:43.230605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:48.257508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:51.071982 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 18:38:53.356294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:38:58.387068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:03.405840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:08.429707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:13.448162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:18.470074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:23.508391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:24.941542 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 18:39:28.548050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:33.568388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:38.609021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:43.628597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:48.647590 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:53.737382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:39:58.757140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:03.866555 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:08.884052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:13.915062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:18.937640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:24.220460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:29.235709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:34.263238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:39.281684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:44.310912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:49.325358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:54.386225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:40:59.403882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:04.433144 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:09.453974 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:14.475249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:19.494177 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:24.565211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:29.589750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:34.616061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:39.640190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:44.677766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:49.697260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:54.726110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:41:59.742075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:04.767403 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:09.790739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:14.814113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:19.837176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:24.870808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:29.886925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:34.910862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:37.317253 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 18:42:39.932841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:44.957886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:49.972943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:42:54.995173 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:00.035607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:05.057007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:10.086466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:15.111147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:20.135918 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:25.163677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:30.181969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:35.203472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:40.224193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:45.247128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:50.270117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:43:55.333727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:00.441312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:05.460698 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:10.480629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:15.502542 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:20.522838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:25.599762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:30.621113 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:35.717720 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:40.740407 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:45.766251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:50.787727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:44:55.807502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:00.823250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:05.848137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:10.866591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:15.886781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:20.904329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:22.094426 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 18:45:25.989811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:31.006822 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:36.023486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:41.039077 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:46.075951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:51.089316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:45:56.149032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:01.198020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:06.215829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:11.233788 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:16.255468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:21.277105 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:26.298616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:31.317991 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:36.335263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:41.359692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:46.377686 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:51.394304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:46:56.411039 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:00.080611 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 18:47:01.434642 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:06.452110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:11.478529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:14.539885 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:47:14.539887 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 18:47:14.539971 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 18:47:14.539974 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 18:47:14.539985 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 18:47:14.539993 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:47:14.539994 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 18:47:14.539999 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:47:14.540007 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:47:14.540007 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 18:47:14.540013 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:47:14.540016 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 18:47:14.540021 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:47:14.540021 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 18:47:14.540029 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 18:47:14.540029 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:47:14.540035 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 18:47:14.540036 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:47:14.540041 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 18:47:14.540044 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:47:14.540054 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:47:14.540059 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:47:14.539986 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 18:47:14.540068 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 18:47:14.540072 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 18:47:14.540080 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 18:47:14.540084 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:47:14.540089 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:47:14.540105 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 18:47:14.540131 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 18:47:14.540138 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 18:47:14.540145 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 18:47:14.540151 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:47:14.540157 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:47:14.540183 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 18:47:14.540195 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 18:47:14.540200 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 18:47:14.540060 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 18:47:14.540645 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 18:47:14.540668 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 18:47:14.540675 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 18:47:14.540680 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 18:47:14.540685 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 18:47:14.540689 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 18:47:14.540068 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:47:14.540712 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:47:14.540718 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:47:14.540723 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:47:14.540729 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 18:47:14.540738 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 18:47:14.540741 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 18:47:14.540745 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 18:47:14.540752 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 18:47:14.540756 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 18:47:14.540761 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 18:47:14.540765 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 18:47:14.540772 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 18:47:14.540779 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 18:47:14.540784 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 18:47:14.540790 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 18:47:14.540805 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 18:47:14.540809 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 18:47:14.540813 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 18:47:14.540816 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 18:47:14.595731 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 18:47:14.595857 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 18:47:14.595926 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 18:47:14.595963 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 18:47:14.595985 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 18:47:14.596010 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 18:47:14.596154 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 18:47:16.516992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:21.539466 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:26.556036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:31.573562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:36.595728 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:41.615880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:46.641768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:51.659467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:47:56.920526 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:01.948085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:07.009347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:12.027757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:17.055471 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:22.087109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:27.110448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:32.129140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:37.165031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:42.182628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:42.336308 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 18:48:47.209031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:52.256222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:48:57.439628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:02.465215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:04.979290 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 2 items received I0617 18:49:07.483646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:12.506342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:17.524831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:22.559816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:27.813721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:32.844178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:37.867426 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:42.888185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:47.911152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:52.949367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:49:57.967779 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:02.991594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:08.009340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:13.039002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:18.088264 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:23.155685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:28.209834 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:33.226086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:38.250337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:43.276773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:48.313322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:53.387233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:50:58.410138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:03.442054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:08.464663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:13.501372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:18.525117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:23.733133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:28.752114 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:33.776200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:38.797007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:43.815542 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:48.843896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:53.967572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:51:58.989520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:04.018455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:09.037858 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:14.060825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:19.095180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:24.116903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:29.137546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:32.102455 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 18:52:34.156850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:39.171927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:44.188929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:49.207517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:54.229736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:52:59.253092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:04.281275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:09.546305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:14.572195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:19.590351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:24.612319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:29.644288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:34.665637 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:39.690496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:44.709158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:49.729124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:54.745782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:53:59.766574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:04.787341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:09.811769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:14.827827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:19.106097 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 18:54:19.847800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:24.886142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:29.911644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:34.925881 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:39.948778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:44.965213 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:49.986684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:54:55.038477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:00.063064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:05.090424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:10.110669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:15.134535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:20.161926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:25.187609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:30.216831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:35.241188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:40.259434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:45.279094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:50.296760 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:55:55.321995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:00.344924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:05.365520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:10.386902 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:15.409310 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:20.499727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:25.562724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:30.586161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:35.614539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:40.651042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:45.670282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:50.694761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:56:55.714765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:00.740438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:05.759243 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:10.780145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:15.347646 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 18:57:15.799301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:20.828201 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:25.853544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:26.982752 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 18:57:30.873631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:35.891640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:40.934828 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:45.968827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:50.993233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:57:56.032261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:01.053209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:06.068192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:11.091010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:16.110615 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:21.129244 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:26.558940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:31.583815 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:36.613582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:41.628095 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:46.654091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:51.678024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:58:56.729725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:01.901458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:06.928485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:11.955733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:16.971808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:19.119858 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 18:59:22.030061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:27.047296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:32.078310 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:37.093732 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:42.121943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:47.146072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:52.170492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 18:59:57.240915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:02.944582 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:07.967544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:12.997363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:18.027684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:23.067166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:28.175793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:33.217277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:38.269301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:43.325879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:48.358947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:53.459755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:00:58.479981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:03.501703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:08.591899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:13.648971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:18.680633 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:23.766448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:28.798792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:33.823975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:38.850056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:43.868553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:48.904118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:53.969120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:01:58.992803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:04.042700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:09.069110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:14.092575 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:14.540081 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:02:14.540177 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:02:14.540283 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 19:02:14.540361 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 19:02:14.540398 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 19:02:14.540463 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 19:02:14.540502 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.540606 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.540678 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:02:14.540730 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 19:02:14.540743 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 19:02:14.540757 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 19:02:14.540820 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 19:02:14.540849 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:02:14.540909 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:02:14.541006 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 19:02:14.541029 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 19:02:14.541036 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 19:02:14.540574 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 19:02:14.541568 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 19:02:14.541581 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 19:02:14.541594 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 19:02:14.541668 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 19:02:14.541679 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 19:02:14.541687 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 19:02:14.541696 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 19:02:14.541623 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 19:02:14.541732 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 19:02:14.541741 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 19:02:14.541754 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 19:02:14.541759 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 19:02:14.541764 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 19:02:14.541769 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 19:02:14.540586 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 19:02:14.541818 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 19:02:14.541827 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.541832 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.541840 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:02:14.541850 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 19:02:14.541856 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 19:02:14.541865 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 19:02:14.541897 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 19:02:14.541904 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:02:14.541912 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:02:14.541970 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 19:02:14.541982 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 19:02:14.541989 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 19:02:14.541998 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 19:02:14.541701 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 19:02:14.542117 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 19:02:14.542142 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 19:02:14.542168 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.542211 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.542240 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:02:14.542285 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 19:02:14.542344 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 19:02:14.542394 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 19:02:14.542439 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 19:02:14.542464 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:02:14.542486 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:02:14.542535 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 19:02:14.542561 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 19:02:14.542589 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 19:02:14.542618 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 19:02:14.600060 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 19:02:14.600150 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 19:02:14.600194 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 19:02:14.600761 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 19:02:14.600856 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 19:02:14.600883 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 19:02:14.600705 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 19:02:14.601048 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 19:02:14.601149 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 19:02:14.601194 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.601232 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:02:14.601269 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:02:14.601317 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 19:02:14.601330 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 19:02:14.601344 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 19:02:14.601465 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 19:02:14.601504 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:02:14.601549 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:02:14.601598 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 19:02:14.601647 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 19:02:14.601671 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 19:02:14.689341 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 19:02:14.689483 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 19:02:14.689559 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 19:02:14.689591 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 19:02:14.689609 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 19:02:14.689648 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 19:02:14.689881 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 19:02:19.120423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:24.145542 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:29.281856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:34.310035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:39.333822 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:44.363224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:49.383931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:54.407716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:02:59.425899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:04.450679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:09.504649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:14.537805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:19.552593 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:24.575743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:29.589040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:30.366823 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 19:03:34.616509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:39.632319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:44.655679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:49.673205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:54.695986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:03:57.416458 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 5 items received I0617 19:03:59.717084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:04.744985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:09.764204 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:14.802821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:19.832769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:19.989371 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 19:04:20.130711 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 6 items received I0617 19:04:24.862197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:29.881895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:34.907452 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:39.947381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:44.969060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:49.989488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:04:55.026133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:00.049212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:05.070256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:10.093725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:15.112984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:20.131273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:25.349384 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:30.369699 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:35.397980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:40.415033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:45.440978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:50.460208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:05:55.483393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:00.504330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:05.529896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:10.551054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:15.580066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:20.595064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:25.619252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:30.635505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:35.660502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:40.686444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:45.708737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:50.734114 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:06:55.813146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:00.834535 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:05.866632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:10.891519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:15.912973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:20.933744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:25.985880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:31.005656 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:36.022646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:41.065040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:46.083661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:51.105305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:07:56.130488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:01.150837 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:06.168370 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:11.198649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:16.249212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:21.266940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:26.293802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:31.309463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:36.428427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:41.453185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:46.479786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:51.499421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:08:56.517953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:01.536404 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:06.557742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:11.575482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:16.600171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:21.616677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:26.137537 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 19:09:26.795330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:31.813965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:36.830922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:41.859193 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:46.882743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:51.902436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:09:57.516912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:02.543169 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:07.569088 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:12.584873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:17.606447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:22.652328 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:27.669541 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:32.684015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:37.704513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:42.719209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:43.010535 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 19:10:47.745392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:52.762032 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:10:57.781368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:02.799044 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:07.441592 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 2 items received I0617 19:11:07.884580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:12.906128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:17.924994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:22.942794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:27.981863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:33.004733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:38.028900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:43.049954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:48.069392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:53.091412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:11:58.116687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:03.134248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:08.168475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:13.198412 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:15.386719 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 19:12:18.218966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:23.321429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:28.356135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:33.387752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:38.417882 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:43.441816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:48.478406 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:53.714711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:12:58.735291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:03.759941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:08.781249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:13.821209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:18.851254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:24.231777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:29.253967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:34.272123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:39.293784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:44.316299 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:49.341872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:54.385765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:13:59.408372 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:04.441285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:09.459053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:14.477916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:19.497854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:24.527393 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:29.545721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:34.580366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:39.599057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:44.626303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:49.646272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:54.667189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:14:59.686641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:04.707500 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:09.725436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:14.742408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:19.765865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:24.782140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:29.799234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:34.815797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:39.843308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:44.857284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:49.879441 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:54.904776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:15:59.933569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:04.949869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:09.981377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:14.994940 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:20.011831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:25.192904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:30.210275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:31.447127 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 19:16:35.228687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:40.256097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:45.288680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:50.311063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:16:55.351679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:00.370395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:05.391186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:10.407721 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:14.541523 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:17:14.541597 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:17:14.541624 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 19:17:14.541638 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 19:17:14.541648 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 19:17:14.541653 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 19:17:14.541661 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.541667 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.541676 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:17:14.541687 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 19:17:14.541693 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 19:17:14.541703 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 19:17:14.541714 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 19:17:14.541719 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:17:14.541727 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:17:14.541774 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 19:17:14.541786 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 19:17:14.541792 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 19:17:14.541801 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 19:17:14.541816 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 19:17:14.541821 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 19:17:14.541826 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.541830 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.541834 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:17:14.541842 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 19:17:14.541847 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 19:17:14.541854 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 19:17:14.541862 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 19:17:14.541867 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:17:14.541872 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:17:14.541886 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 19:17:14.541900 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 19:17:14.541905 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 19:17:14.542334 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 19:17:14.542347 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 19:17:14.542354 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 19:17:14.542364 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 19:17:14.542370 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 19:17:14.542375 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 19:17:14.542380 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 19:17:14.542385 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 19:17:14.542388 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 19:17:14.542403 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 19:17:14.542410 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 19:17:14.542417 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 19:17:14.542420 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 19:17:14.542423 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 19:17:14.542426 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 19:17:14.542429 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 19:17:14.542435 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 19:17:14.542438 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 19:17:14.542443 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.542446 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.542451 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:17:14.542457 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 19:17:14.542460 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 19:17:14.542465 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 19:17:14.542470 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 19:17:14.542473 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:17:14.542479 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:17:14.542508 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 19:17:14.542513 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 19:17:14.542517 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 19:17:14.542523 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 19:17:14.608070 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 19:17:14.608144 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 19:17:14.608182 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 19:17:14.608213 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 19:17:14.608231 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 19:17:14.608261 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 19:17:14.608304 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 19:17:14.608325 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 19:17:14.608345 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.608366 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:17:14.608389 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:17:14.608410 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 19:17:14.608428 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 19:17:14.608455 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 19:17:14.608480 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 19:17:14.608498 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:17:14.608522 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:17:14.608559 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 19:17:14.608597 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 19:17:14.608617 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 19:17:14.608968 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 19:17:14.651121 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 19:17:14.651235 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 19:17:14.651295 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 19:17:14.651322 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 19:17:14.651392 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 19:17:14.651421 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 19:17:14.651511 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 19:17:15.432451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:20.457687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:25.562315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:30.578903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:35.605385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:40.624166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:45.645669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:50.662562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:17:56.179951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:00.142989 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 10 items received I0617 19:18:01.031001 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 19:18:01.202711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:06.242493 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:11.259234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:16.279441 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:21.295244 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:26.360401 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:31.374996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:36.389923 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:41.406630 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:46.423242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:51.437435 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:18:56.459936 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:01.560503 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:06.586585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:11.603084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:16.620767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:21.637038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:26.933804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:31.968654 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:36.989731 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:42.010802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:47.068898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:52.088456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:19:57.339640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:02.364020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:07.381629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:12.400586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:17.420971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:22.439758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:27.459442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:32.475419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:36.402688 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 19:20:37.492532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:42.512318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:47.527528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:52.543492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:20:57.565602 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:02.588221 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:07.618019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:12.635690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:17.649998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:22.665574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:27.849305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:32.865097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:37.890947 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:42.909735 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:43.455916 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 19:21:47.929287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:52.949803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:21:57.982971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:03.001932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:08.017982 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:13.037611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:18.055981 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:23.073846 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:28.089529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:33.495070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:38.511617 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:43.537476 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:48.558029 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:54.003830 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:22:59.024487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:04.047139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:09.066836 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:14.093250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:19.115712 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:24.145910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:29.165368 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:34.184180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:39.203273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:44.232946 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:49.262063 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:54.317745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:23:59.334707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:04.357634 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:09.376106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:14.391244 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:19.415650 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:24.453815 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:29.489487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:34.510003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:39.525047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:44.540866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:45.146411 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 19:24:49.555716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:54.572529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:24:59.599156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:04.631010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:09.655230 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:14.669042 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:19.691339 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:25.094138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:30.109692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:35.131152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:40.150361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:45.171729 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:50.187653 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:25:55.366095 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:00.383121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:05.402399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:10.420971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:15.438464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:20.451348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:25.497881 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:30.522819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:35.546093 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:40.567281 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:45.590958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:50.611967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:26:55.654359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:00.745172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:05.767040 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:10.783354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:15.815225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:20.831159 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:26.001740 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:31.033728 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:36.058616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:41.072660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:46.098703 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:48.038115 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 9 items received I0617 19:27:51.125054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:27:56.143963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:01.162651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:06.190215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:11.212824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:16.240253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:21.263156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:26.281171 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:31.305815 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:36.410203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:41.479442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:46.503744 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:51.529226 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:28:56.565388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:01.591462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:06.612871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:11.650375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:16.674291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:21.689460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:26.753812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:28.418737 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 19:29:31.774374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:36.811705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:41.828598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:46.857031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:51.876298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:29:56.989748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:02.111890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:07.131913 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:12.150594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:17.182455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:22.201068 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:27.221738 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:32.255175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:37.272349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:42.292969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:47.315036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:52.353543 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:30:57.372097 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:02.392046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:07.412792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:12.435800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:17.455340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:18.461044 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 19:31:22.475843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:27.529759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:32.616825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:37.637425 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:42.666647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:47.692445 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:52.715234 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:31:57.735823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:02.763892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:07.780922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:12.799705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:14.542707 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:32:14.543016 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 19:32:14.542649 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:32:14.543279 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 19:32:14.543361 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 19:32:14.543445 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 19:32:14.543527 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 19:32:14.543618 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 19:32:14.543671 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 19:32:14.543746 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 19:32:14.543812 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 19:32:14.543867 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 19:32:14.543957 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 19:32:14.544027 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 19:32:14.544105 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 19:32:14.544161 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 19:32:14.544229 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 19:32:14.544282 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 19:32:14.544355 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 19:32:14.544442 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 19:32:14.544512 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 19:32:14.544587 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 19:32:14.544665 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:32:14.544723 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:32:14.544795 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 19:32:14.544880 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 19:32:14.544961 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 19:32:14.545025 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 19:32:14.545101 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:32:14.545178 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:32:14.545293 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 19:32:14.545370 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 19:32:14.545454 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 19:32:14.543127 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 19:32:14.546001 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 19:32:14.546183 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 19:32:14.546268 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 19:32:14.546346 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:32:14.546439 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:32:14.546672 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 19:32:14.546794 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 19:32:14.546880 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 19:32:14.546965 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 19:32:14.547048 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:32:14.547139 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:32:14.547441 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 19:32:14.547542 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 19:32:14.547621 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 19:32:14.547765 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 19:32:14.611157 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 19:32:14.611411 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 19:32:14.611554 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 19:32:14.612003 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 19:32:14.612135 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 19:32:14.612212 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 19:32:14.611627 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 19:32:17.817640 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:22.835054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:27.865285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:32.988969 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:38.008273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:43.034247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:48.064532 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:53.113985 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:32:58.136522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:03.167922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:08.188442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:13.291207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:18.334138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:23.402816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:28.421312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:33.444168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:38.463109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:43.488659 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:48.507112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:53.540023 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:33:58.557737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:03.151125 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 4 items received I0617 19:34:03.574268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:08.598083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:13.623793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:18.641508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:23.706610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:28.730470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:33.750386 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:38.767527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:43.789468 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:48.807103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:53.931762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:34:58.957569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:03.975910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:08.996473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:14.018278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:19.083949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:24.131317 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:29.151387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:34.210461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:39.227702 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:44.247184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:49.264711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:54.442696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:35:59.475364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:04.045933 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 19:36:04.491006 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:09.508517 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:14.521545 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:19.542267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:24.764377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:29.791076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:34.906598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:39.939977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:44.960195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:49.994384 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:36:55.021781 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:00.065206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:05.083982 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:10.112786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:15.128853 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:20.166762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:25.185708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:30.209529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:35.223922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:40.243716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:45.261488 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:50.288387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:37:55.304106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:00.323225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:05.338004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:10.358806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:15.381169 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:20.405127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:25.426782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:30.446886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:35.474959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:40.491725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:45.507305 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:46.425587 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 19:38:50.525181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:38:55.685262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:00.721124 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:05.733989 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:10.806990 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:15.821644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:20.843636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:25.859133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:30.878892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:35.899580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:40.913319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:45.938269 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:50.955973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:39:55.990135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:01.006461 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:04.487148 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 10 items received I0617 19:40:06.048819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:11.068170 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:16.082957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:21.097847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:26.230240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:31.251933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:36.272240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:41.290939 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:46.325539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:51.351903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:40:56.382207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:01.421091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:06.443206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:11.460935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:16.487971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:21.518229 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:26.536064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:31.562495 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:36.581827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:41.620872 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:46.639261 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:51.659363 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:41:56.703286 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:01.774002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:06.788648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:11.806843 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:16.824322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:21.838169 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:26.852568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:31.887150 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:36.917893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:42.032544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:47.048390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:52.069956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:57.096742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:42:57.163483 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 19:43:02.143799 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:07.169998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:12.185215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:17.226762 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:22.248992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:27.267986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:32.303263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:37.322605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:42.351033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:47.373531 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:52.394189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:43:57.417483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:02.473562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:07.496094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:12.514645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:17.532997 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:22.556290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:27.570980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:32.610373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:37.625867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:42.646131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:47.661730 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:52.681540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:44:57.703829 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:02.725293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:07.748025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:09.057155 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 5 items received I0617 19:45:12.765360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:17.793253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:22.821242 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:28.107376 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:33.154566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:38.175658 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:43.203724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:48.222439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:53.351421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:45:58.375081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:03.393971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:08.426200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:13.566949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:18.585956 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:23.951266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:28.969357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:33.990331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:39.012527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:44.030505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:49.048912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:54.094341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:46:59.119718 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:04.146926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:09.165089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:14.187374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:14.543751 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:47:14.543827 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 19:47:14.543841 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 19:47:14.543852 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 19:47:14.543858 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 19:47:14.543867 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.543875 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.543883 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:47:14.543892 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 19:47:14.543899 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 19:47:14.543909 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 19:47:14.543919 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 19:47:14.543924 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:47:14.543933 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:47:14.543985 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 19:47:14.543994 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 19:47:14.544001 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 19:47:14.544009 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 19:47:14.544025 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 19:47:14.544030 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 19:47:14.544036 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.544040 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.544045 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:47:14.544052 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 19:47:14.544057 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 19:47:14.544066 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 19:47:14.544074 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 19:47:14.544084 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:47:14.544089 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:47:14.544102 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 19:47:14.544114 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 19:47:14.544118 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 19:47:14.544655 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 19:47:14.544691 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 19:47:14.544701 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 19:47:14.544710 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 19:47:14.544719 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 19:47:14.544727 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 19:47:14.544736 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 19:47:14.544742 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 19:47:14.544748 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 19:47:14.544752 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 19:47:14.544767 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 19:47:14.544774 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 19:47:14.544781 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 19:47:14.544785 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 19:47:14.544790 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 19:47:14.544794 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 19:47:14.544798 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 19:47:14.544816 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 19:47:14.544821 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 19:47:14.544830 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.544834 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.544841 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:47:14.544849 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 19:47:14.544854 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 19:47:14.544863 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 19:47:14.544871 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 19:47:14.544876 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:47:14.544883 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:47:14.544920 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 19:47:14.544929 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 19:47:14.544935 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 19:47:14.544941 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 19:47:14.614825 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 19:47:14.615101 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 19:47:14.615242 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 19:47:14.615767 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 19:47:14.615911 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 19:47:14.615997 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 19:47:14.616098 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 19:47:14.616181 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 19:47:14.616272 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.615369 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 19:47:14.616449 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 19:47:14.616508 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 19:47:14.616580 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 19:47:14.616657 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 19:47:14.616730 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 19:47:14.616813 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 19:47:14.616869 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 19:47:14.616941 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 19:47:14.617033 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 19:47:14.617128 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 19:47:14.617199 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 19:47:14.678906 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 19:47:14.679206 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 19:47:14.679298 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 19:47:14.679311 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 19:47:14.679367 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 19:47:14.679433 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 19:47:14.679850 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 19:47:18.440969 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 19:47:19.205121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:24.229161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:29.251275 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:34.280986 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:39.296178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:44.319015 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:49.339429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:54.376526 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:47:59.404784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:04.423306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:09.440507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:14.458737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:19.473852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:24.505440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:29.523705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:34.541754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:39.563342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:44.587329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:49.606806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:54.957719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:48:56.492792 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 9 items received I0617 19:48:59.982259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:04.999943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:10.024483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:15.039768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:20.058675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:25.084241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:30.118043 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:35.139256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:40.155147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:45.178344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:50.195081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:49:55.212750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:00.231117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:05.252180 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:10.274236 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:15.295496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:20.319614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:25.366547 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:30.386745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:35.408484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:40.421813 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:45.455188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:50.479848 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:50:55.499708 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:00.520501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:05.543972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:10.581808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:15.607484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:19.068179 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 19:51:20.623447 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:25.652262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:27.213990 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 19:51:30.682267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:35.699871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:40.719591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:45.735520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:50.755667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:51:55.771156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:00.786842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:05.803202 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:10.825511 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:15.848776 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:20.864046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:25.904789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:30.929803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:35.961943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:40.975496 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:45.999443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:51.019160 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:52:56.059191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:01.078058 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:06.099693 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:11.121140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:16.139608 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:21.152443 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:26.180122 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:31.199374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:36.227233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:41.245329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:46.271623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:51.291850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:53:56.315122 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:01.366823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:06.385576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:11.403380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:16.420358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:21.440057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:26.544897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:31.663489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:36.686988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:41.704115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:46.724369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:51.740880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:54:56.757854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:01.777900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:06.798799 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:11.817951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:16.838632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:21.866652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:26.889312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:31.905111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:36.930558 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:41.958586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:46.988125 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:52.080154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:55:57.318601 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:02.332523 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:07.408159 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:12.424767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:17.445595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:22.460395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:27.807285 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:30.446865 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 11 items received I0617 19:56:32.826065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:37.858303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:42.876027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:47.898820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:52.916219 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:56:58.015739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:03.040685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:08.057505 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:13.074568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:18.098480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:23.131800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:28.155784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:33.177117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:38.209294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:43.238607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:48.263859 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:53.334933 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:57:58.358066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:03.380302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:08.403771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:13.435786 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:18.465865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:23.498209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:28.517268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:33.613701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:34.077244 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 19:58:34.515334 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 8 items received I0617 19:58:38.639864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:43.655652 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:48.673798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:53.719344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:58:58.738599 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:03.760553 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:08.779695 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:13.794433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:18.820842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:23.917137 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:28.942018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:33.959341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:38.980460 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:44.001987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:49.023406 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:54.454238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 19:59:59.469441 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:04.490168 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:09.516550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:14.534696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:17.278468 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 9 items received I0617 20:00:19.550237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:24.572663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:29.590001 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:34.613726 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:39.646374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:44.668874 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:49.694139 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:54.719555 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:00:59.790487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:04.810224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:09.844941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:14.865335 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:19.897238 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:24.929329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:29.962489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:34.976791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:39.997072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:45.024700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:50.050825 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:01:55.090385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:00.108948 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:05.126057 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:10.145253 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:14.545464 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:02:14.545549 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 20:02:14.545565 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 20:02:14.545575 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:02:14.545581 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:02:14.545589 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.545595 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.545624 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:02:14.545703 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:02:14.545712 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:02:14.545818 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:02:14.545840 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:02:14.545846 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:02:14.545856 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:02:14.545922 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:02:14.545985 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:02:14.545996 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:02:14.546069 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 20:02:14.546111 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 20:02:14.546118 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 20:02:14.546169 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.546177 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.546184 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:02:14.546192 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 20:02:14.546258 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 20:02:14.546269 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 20:02:14.546329 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 20:02:14.546339 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:02:14.546345 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:02:14.546417 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 20:02:14.546431 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 20:02:14.546436 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 20:02:14.546935 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:02:14.546983 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 20:02:14.546999 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 20:02:14.547008 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 20:02:14.547009 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 20:02:14.547016 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 20:02:14.547018 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 20:02:14.547026 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 20:02:14.547026 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 20:02:14.547037 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 20:02:14.547039 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 20:02:14.547044 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 20:02:14.547051 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 20:02:14.547055 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 20:02:14.547044 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 20:02:14.547077 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 20:02:14.547081 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 20:02:14.547099 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:02:14.547106 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:02:14.547112 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.547116 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.547122 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:02:14.547129 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:02:14.547134 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:02:14.547140 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:02:14.547161 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:02:14.547164 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:02:14.547171 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:02:14.547196 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:02:14.547200 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:02:14.547204 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:02:14.547208 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 20:02:14.615030 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 20:02:14.615053 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 20:02:14.615081 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 20:02:14.615090 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 20:02:14.615095 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 20:02:14.615107 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 20:02:14.615138 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 20:02:14.615173 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 20:02:14.615195 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.615212 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:02:14.615254 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:02:14.615744 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 20:02:14.615857 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 20:02:14.615898 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 20:02:14.616005 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 20:02:14.616099 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:02:14.616161 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:02:14.616183 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 20:02:14.616201 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 20:02:14.616268 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 20:02:14.615690 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 20:02:14.656926 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 20:02:14.656951 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 20:02:14.656979 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 20:02:14.656989 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 20:02:14.656996 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 20:02:14.657007 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 20:02:14.657145 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 20:02:15.166329 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:20.183158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:25.234369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:30.250007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:35.266316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:40.286580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:45.306294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:50.324572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:02:52.454953 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 20:02:55.370332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:00.393069 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:05.414091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:10.434849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:15.449530 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:20.488053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:25.552081 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:30.575456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:35.595360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:40.618845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:45.656126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:50.522242 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 3 items received I0617 20:03:50.672980 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:03:55.690118 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:00.710302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:05.752951 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:10.775977 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:15.794293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:20.813966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:25.925751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:30.948473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:35.973292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:40.988128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:46.007978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:51.026188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:04:56.051660 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:01.070646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:06.091200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:11.106423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:16.129726 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:21.154442 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:26.365172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:31.383378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:36.401647 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:41.452596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:46.475927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:47.085724 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 4 items received I0617 20:05:51.495696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:05:56.517759 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:01.538521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:06.551679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:11.566790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:16.583821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:21.602516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:26.618958 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:31.636240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:36.656268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:41.671323 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:46.685140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:51.700801 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:06:57.057671 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:02.074008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:07.096396 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:12.114487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:17.134551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:22.151795 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:23.288993 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 6 items received I0617 20:07:27.178528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:32.204379 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:37.217302 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:42.234152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:47.256510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:52.278258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:07:57.359817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:02.379157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:07.396268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:12.410050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:17.438073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:22.457670 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:27.593791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:32.612292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:37.629066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:42.648924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:47.667456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:52.686972 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:08:57.713808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:02.736369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:07.756436 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:08.467618 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 20:09:12.775629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:17.795131 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:22.811525 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:27.831371 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:32.859272 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:37.892290 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:42.928904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:47.970621 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:52.991619 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:09:58.011432 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:03.047895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:08.068932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:13.084233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:18.106757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:23.138064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:28.153295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:33.180146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:38.208598 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:43.231202 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:48.251522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:53.405190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:10:58.436487 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:03.459639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:08.482456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:13.501962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:18.564282 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:23.888135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:28.933055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:33.948493 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:38.984367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:44.006381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:46.535943 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 20:11:49.032645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:54.071288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:11:59.109459 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:04.130454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:09.152963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:14.183011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:19.207649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:24.276657 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:29.298509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:34.317046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:39.335279 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:44.355356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:49.375380 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:54.393628 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:12:59.416965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:04.432061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:09.448332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:14.469632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:19.491483 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:24.522279 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:29.553178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:34.576674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:39.594606 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:41.299830 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 20:13:44.615087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:46.111047 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 20:13:49.644115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:54.664294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:13:59.683326 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:04.705148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:09.726998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:14.746870 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:19.771942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:24.797151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:29.813554 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:34.853724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:39.866132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:44.882574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:49.908051 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:14:55.152439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:00.171233 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:05.215787 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:10.234914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:15.247654 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:20.269263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:25.285314 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:30.302697 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:35.317865 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:40.332034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:45.362467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:50.377259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:15:55.464065 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:00.486181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:05.503414 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:10.527540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:15.545337 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:20.562868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:25.647364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:30.666491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:35.690444 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:40.712646 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:45.733375 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:50.750248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:16:55.778692 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:00.796448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:05.817298 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:10.837855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:14.545653 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:17:14.545746 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 20:17:14.545763 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 20:17:14.545773 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:17:14.545779 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:17:14.545786 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.545793 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.545796 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 20:17:14.545800 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:17:14.545804 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 20:17:14.545811 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.545834 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.545811 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:17:14.545840 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:17:14.545843 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:17:14.545847 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 20:17:14.545852 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 20:17:14.545855 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:17:14.545860 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 20:17:14.545866 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:17:14.545868 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 20:17:14.545871 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:17:14.545873 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:17:14.545879 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:17:14.545936 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 20:17:14.545961 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 20:17:14.545968 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 20:17:14.545879 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:17:14.546014 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:17:14.546021 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:17:14.546026 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:17:14.546034 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 20:17:14.547097 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:17:14.547192 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 20:17:14.547211 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 20:17:14.547218 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 20:17:14.547231 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 20:17:14.547241 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 20:17:14.547248 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 20:17:14.547254 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 20:17:14.547262 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 20:17:14.547267 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 20:17:14.547219 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 20:17:14.547292 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 20:17:14.547298 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 20:17:14.547301 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:17:14.547303 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 20:17:14.547307 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 20:17:14.547307 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:17:14.547314 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 20:17:14.547315 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.547319 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 20:17:14.547320 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.547326 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:17:14.547336 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:17:14.547341 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:17:14.547348 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:17:14.547355 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:17:14.547358 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:17:14.547371 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:17:14.547564 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:17:14.547569 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:17:14.547574 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:17:14.547579 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 20:17:14.607855 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 20:17:14.608124 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 20:17:14.608272 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 20:17:14.608759 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 20:17:14.608876 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 20:17:14.608968 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 20:17:14.609107 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 20:17:14.609194 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 20:17:14.609209 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.609215 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:17:14.609240 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:17:14.609270 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 20:17:14.609276 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 20:17:14.608426 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 20:17:14.609339 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 20:17:14.609853 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 20:17:14.609938 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:17:14.610034 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:17:14.610124 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 20:17:14.610218 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 20:17:14.610229 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 20:17:14.649375 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 20:17:14.649476 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 20:17:14.649543 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 20:17:14.649568 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 20:17:14.649605 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 20:17:14.649633 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 20:17:14.649711 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 20:17:15.997458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:21.012745 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:26.037508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:31.055784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:36.086474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:41.101175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:46.117917 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:51.135433 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:17:56.159140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:01.182636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:06.204212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:11.233117 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:16.255037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:21.275793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:26.329714 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:31.382883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:36.403250 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:41.425151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:46.439133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:51.466772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:18:52.474111 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 1 items received I0617 20:18:56.481812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:01.587106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:06.612413 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:11.632278 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:16.655893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:21.684304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:27.002849 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:32.024524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:37.044228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:42.249078 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:47.270867 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:50.115928 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 20:19:52.285852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:19:57.320486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:02.346222 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:07.364212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:12.382890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:17.407392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:17.550640 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 9 items received I0617 20:20:22.434231 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:25.336858 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 20:20:27.546152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:32.568973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:37.594293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:42.623202 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:47.651988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:52.675854 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:20:57.696172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:02.718807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:07.739635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:12.755432 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:17.771932 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:22.791871 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:27.809295 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:32.836420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:37.855610 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:42.872612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:47.896017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:52.919540 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:21:57.937047 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:03.060613 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:08.075845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:13.094513 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:18.114411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:23.136663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:28.172196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:33.192717 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:38.210888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:43.252822 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:48.296510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:53.334366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:22:58.348313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:03.367155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:08.414225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:13.440784 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:18.456573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:23.542094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:28.560861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:33.581076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:38.686062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:43.704304 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:48.750726 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:53.870742 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:23:58.896183 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:03.920325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:08.949688 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:13.966030 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:19.074851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:24.101941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:29.157924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:34.177508 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:39.195389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:44.214414 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:49.240189 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:54.255478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:24:59.274521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:04.292035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:09.310148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:14.329260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:19.344309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:24.363845 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:29.395112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:34.421911 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:39.441080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:42.495393 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 20:25:44.465354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:49.480700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:54.497021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:25:59.524240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:04.546724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:09.557561 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 20:26:09.569890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:14.584886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:19.638724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:24.656672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:29.698792 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:34.712704 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:35.411722 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 5 items received I0617 20:26:39.733797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:44.750857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:49.769031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:54.786821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:26:59.807356 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:04.833156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:09.856151 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:14.878775 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:19.893910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:24.947415 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:29.968734 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:33.129621 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 20:27:34.990651 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:40.007893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:45.029456 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:50.045467 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:27:55.201777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:00.236953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:05.258312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:10.275916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:15.291004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:20.326266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:25.347915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:30.367754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:35.385641 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:40.399680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:45.419480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:50.436826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:28:55.450966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:00.466501 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:05.478889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:10.499209 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:15.510400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:20.526366 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:25.564586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:30.586167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:35.612783 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:40.636110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:45.663434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:50.680944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:29:55.992663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:01.019004 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:06.044423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:11.065790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:16.084752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:21.103381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:26.123265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:31.136423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:36.153025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:41.174509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:46.208529 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:51.250387 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:30:56.434033 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:01.453919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:06.467492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:11.483805 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:16.500120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:21.517737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:26.530516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:31.545594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:36.567185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:41.583642 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:46.602994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:51.619942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:31:56.706048 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:01.731175 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:06.752898 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:11.777262 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:14.555473 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:32:14.555951 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:32:14.556195 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 20:32:14.556297 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 20:32:14.556364 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 20:32:14.556377 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 20:32:14.556464 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.556478 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.556486 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:32:14.556509 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 20:32:14.556571 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 20:32:14.556607 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 20:32:14.556367 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:32:14.556648 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:32:14.556714 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.556726 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.556732 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:32:14.556684 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 20:32:14.556806 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:32:14.556814 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:32:14.556881 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:32:14.556924 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:32:14.556955 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:32:14.557020 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:32:14.557032 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:32:14.557132 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:32:14.556857 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:32:14.557142 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:32:14.557215 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:32:14.557248 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 20:32:14.557255 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 20:32:14.557288 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 20:32:14.557357 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 20:32:14.556351 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 20:32:14.557534 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 20:32:14.557593 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 20:32:14.557633 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 20:32:14.557685 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 20:32:14.557720 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 20:32:14.557774 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 20:32:14.557818 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 20:32:14.557869 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 20:32:14.557919 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 20:32:14.557977 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 20:32:14.558015 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 20:32:14.558064 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 20:32:14.558210 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 20:32:14.558241 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 20:32:14.558289 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 20:32:14.558347 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:32:14.558401 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:32:14.558422 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.558484 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.558560 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:32:14.558617 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:32:14.558967 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:32:14.559031 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:32:14.559135 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:32:14.559166 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:32:14.559325 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:32:14.559418 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:32:14.559455 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:32:14.559496 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:32:14.559543 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 20:32:14.624423 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 20:32:14.624591 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 20:32:14.624637 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 20:32:14.624685 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 20:32:14.624717 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 20:32:14.624766 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 20:32:14.624817 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 20:32:14.624890 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 20:32:14.624921 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.624978 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:32:14.625008 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:32:14.625061 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 20:32:14.625191 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 20:32:14.625242 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 20:32:14.625301 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 20:32:14.625339 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:32:14.625388 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:32:14.625520 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 20:32:14.625589 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 20:32:14.625618 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 20:32:14.625848 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 20:32:14.667599 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 20:32:14.667716 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 20:32:14.667792 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 20:32:14.667820 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 20:32:14.667872 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 20:32:14.667902 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 20:32:14.668340 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 20:32:16.796575 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:21.816881 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:26.837840 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:31.857560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:36.877458 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:41.918038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:43.575542 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 20:32:46.951755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:51.973519 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:32:57.010155 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:02.042010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:07.070303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:12.087807 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:17.107586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:22.132811 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:27.278827 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:29.431078 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 20:33:32.298184 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:36.137952 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 20:33:37.317961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:42.332087 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:47.354839 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:52.374157 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:33:57.395997 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:02.416550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:07.435217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:12.464067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:17.490925 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:22.513053 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:24.522445 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 8 items received I0617 20:34:27.530907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:32.550546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:37.566382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:42.585793 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:47.602904 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:52.632188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:34:57.657592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:02.676294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:07.704076 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:12.720359 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:17.738727 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:22.756754 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:27.777010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:32.802303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:37.819182 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:42.836240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:47.851724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:52.868595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:35:57.941915 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:02.968639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:07.983176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:13.004895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:18.029245 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:23.056301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:28.074680 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:33.094897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:38.115078 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:43.154808 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:48.177868 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:53.204407 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:36:58.226855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:03.249622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:08.268683 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:13.324894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:18.352968 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:23.994067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:29.010789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:34.042779 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:39.062395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:44.084191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:49.101473 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:54.209761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:37:59.237133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:04.256214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:09.279663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:14.316622 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:19.335251 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:24.369438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:29.388174 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:34.413454 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:39.438572 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:44.454916 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:49.474789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:54.496358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:38:59.509075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:04.526929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:06.446563 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 20:39:09.544422 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:14.560586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:19.583166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:24.605176 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:29.624012 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:34.644395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:39.661673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:44.691798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:49.713362 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:39:55.257571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:00.280085 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:05.310489 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:10.330830 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:15.355655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:20.371009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:25.401046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:30.418880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:35.438263 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:40.451648 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:45.479684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:50.495154 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:40:51.144639 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 20:40:55.522869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:00.542274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:05.572534 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:10.591758 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:15.610564 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:20.631674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:25.660752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:30.683110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:35.707994 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:40.731446 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:45.746645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:50.763908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:55.782571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:41:57.538976 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 20:42:00.797482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:05.598558 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 10 items received I0617 20:42:05.838907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:10.861927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:15.883357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:20.898459 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:25.924115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:30.939296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:35.976291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:40.993752 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:46.031430 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:51.045334 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:42:56.521031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:01.540110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:06.562766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:11.575655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:16.592521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:21.611733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:26.755481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:31.773349 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:36.802559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:41.820120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:46.837266 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:51.850847 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:43:57.311750 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:02.330536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:07.348470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:12.366274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:17.387569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:22.405481 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:27.423828 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:32.441341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:37.457725 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:42.487115 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:47.505315 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:52.522766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:44:57.539378 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:02.807227 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:07.829391 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:12.849887 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:17.878908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:22.902423 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:27.928424 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:32.950167 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:37.971254 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:42.991694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:48.010161 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:53.030679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:45:58.048445 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:03.071060 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:06.469288 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 2 items received I0617 20:46:08.095567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:13.112903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:18.133998 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:23.214427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:28.246070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:30.150963 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 20:46:33.271689 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:38.295083 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:43.325325 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:48.346205 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:53.469820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:46:58.498676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:03.517355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:08.534267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:13.549029 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:14.557215 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:47:14.557280 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 20:47:14.557305 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 20:47:14.557321 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 20:47:14.557334 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 20:47:14.557349 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 20:47:14.557355 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 20:47:14.557364 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 20:47:14.557376 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 20:47:14.557382 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 20:47:14.557437 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 20:47:14.557472 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 20:47:14.557485 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 20:47:14.557497 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 20:47:14.557506 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 20:47:14.557516 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 20:47:14.557526 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:47:14.557536 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:47:14.557552 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 20:47:14.557564 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 20:47:14.557575 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 20:47:14.557590 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 20:47:14.557595 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:47:14.557604 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:47:14.557650 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 20:47:14.557666 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 20:47:14.557676 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 20:47:14.558057 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:47:14.558075 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:47:14.558082 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:47:14.558087 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:47:14.558093 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:47:14.558102 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:47:14.558109 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:47:14.558120 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:47:14.558141 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:47:14.558152 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:47:14.558160 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:47:14.558192 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:47:14.558200 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:47:14.558208 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:47:14.558225 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 20:47:14.558254 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 20:47:14.558271 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 20:47:14.558285 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 20:47:14.558290 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 20:47:14.558297 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 20:47:14.558307 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 20:47:14.558312 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 20:47:14.558345 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 20:47:14.558354 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 20:47:14.558359 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 20:47:14.558363 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 20:47:14.558369 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 20:47:14.558376 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 20:47:14.558399 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 20:47:14.558413 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 20:47:14.558423 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 20:47:14.558434 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 20:47:14.558441 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 20:47:14.558467 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 20:47:14.558482 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 20:47:14.558491 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 20:47:14.558502 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 20:47:14.629153 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 20:47:14.629178 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 20:47:14.629245 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 20:47:14.629752 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 20:47:14.629848 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 20:47:14.629885 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 20:47:14.629942 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 20:47:18.564336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:23.699123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:28.718607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:33.748962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:38.784005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:43.822824 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:48.843138 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:53.911802 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:47:58.940152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:03.966105 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:08.986309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:14.005755 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:19.044143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:24.072135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:29.093571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:34.111780 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:39.128709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:44.146291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:49.191019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:54.764735 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:48:59.793024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:04.810632 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:09.834664 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:14.853089 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:19.874163 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:24.910166 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:29.937893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:34.956296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:39.995311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:45.020852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:50.045769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:55.064218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:49:55.962533 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 3 items received I0617 20:50:00.083586 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:05.122146 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:10.143971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:15.157543 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:20.181953 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:25.224306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:30.240766 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:35.265976 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:40.305773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:45.352690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:50.374506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:50:55.421860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:00.440431 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:05.467156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:10.491348 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:15.511061 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:20.524101 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:25.551336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:30.573052 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:35.610439 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:40.636203 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:45.655626 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:50.827288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:51:54.615937 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 9 items received I0617 20:51:55.848354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:00.872596 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:01.165188 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 6 items received I0617 20:52:05.891469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:10.915569 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:16.021352 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:21.060866 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:24.482381 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 20:52:26.267397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:31.296474 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:36.317019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:41.335383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:46.356034 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:51.377543 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:52:56.760025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:01.777631 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:06.813389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:11.835712 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:16.856021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:21.876291 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:26.889820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:31.919655 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:36.933634 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:41.958002 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:46.982967 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:52.007961 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:53:57.023790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:02.055789 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:07.071777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:12.086306 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:17.113514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:22.133112 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:27.148271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:32.168340 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:37.213108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:42.235796 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:47.254548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:52.277035 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:54:57.312224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:02.343673 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:07.359392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:12.372502 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:17.386482 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:22.405779 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:27.661693 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:32.679449 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:37.701074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:42.723344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:47.743821 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:52.765568 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:55:57.799110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:02.816237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:07.838643 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:12.858833 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:17.894014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:22.915663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:27.950830 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:32.973841 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:38.022912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:43.040507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:48.056571 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:53.075453 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:56:58.115326 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:03.152591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:08.196644 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:13.221896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:18.245574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:23.328273 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:28.356152 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:33.378749 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:38.397462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:43.418790 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:45.183754 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 20:57:48.433996 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:53.932119 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:57:58.953910 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:03.976663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:09.218395 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:14.244886 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:19.264351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:22.971831 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 20:58:24.281188 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:29.299724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:34.324296 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:39.348618 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:44.363567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:49.384303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:54.411025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:58:59.445576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:04.470612 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:09.488059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:14.518741 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:19.538091 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:24.631791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:29.647892 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:34.670072 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:39.687635 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:44.706469 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:49.732506 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 20:59:55.125675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:00.151385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:05.177421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:10.191411 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:15.210901 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:20.243942 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:25.265929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:29.640915 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 21:00:30.289237 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:35.309367 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:40.335812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:45.358794 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:49.514538 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 21:00:50.566962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:00:55.632406 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:00.647110 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:05.676462 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:10.690347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:15.823025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:20.834889 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:26.285567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:31.302399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:36.447528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:41.463206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:46.481771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:51.510389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:01:56.528294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:01.544414 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:06.563043 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:11.581579 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:14.558109 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:02:14.558337 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 21:02:14.558376 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 21:02:14.558443 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:02:14.558516 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:02:14.558531 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.558537 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.558546 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:02:14.558612 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:02:14.558658 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:02:14.558705 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:02:14.558744 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:02:14.558774 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:02:14.558784 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:02:14.558854 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:02:14.558885 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:02:14.558892 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:02:14.558903 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 21:02:14.558923 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:02:14.558929 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:02:14.558936 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.558941 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.558963 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:02:14.558971 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:02:14.558976 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:02:14.558986 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:02:14.558995 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:02:14.559000 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:02:14.559007 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:02:14.559024 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:02:14.559055 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:02:14.559061 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:02:14.559610 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:02:14.559650 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 21:02:14.559662 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:02:14.559673 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 21:02:14.559684 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 21:02:14.559722 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:02:14.559732 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 21:02:14.559740 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 21:02:14.559752 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 21:02:14.559762 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 21:02:14.559768 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 21:02:14.559792 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 21:02:14.559692 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 21:02:14.559835 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 21:02:14.559841 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 21:02:14.559847 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 21:02:14.559850 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 21:02:14.559896 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:02:14.559903 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:02:14.559909 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.559915 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.559921 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:02:14.559963 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:02:14.559971 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:02:14.559979 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:02:14.559990 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:02:14.559996 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:02:14.560005 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:02:14.560064 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:02:14.560071 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:02:14.560076 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:02:14.560084 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 21:02:14.635204 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:02:14.635229 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:02:14.635256 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:02:14.635267 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:02:14.635272 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:02:14.635284 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:02:14.635399 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:02:14.635406 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:02:14.635436 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.635442 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:02:14.635450 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:02:14.635459 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:02:14.635465 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:02:14.635478 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:02:14.635489 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:02:14.635510 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:02:14.635519 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:02:14.635541 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:02:14.635556 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:02:14.635561 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:02:14.636293 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 21:02:14.713907 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:02:14.713932 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:02:14.713957 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:02:14.713965 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:02:14.713971 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:02:14.713982 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:02:14.714062 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:02:16.598751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:21.618121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:26.637701 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:31.657609 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:36.677851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:41.703259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:46.722415 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:51.736013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:02:57.185965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:02.202457 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:07.222283 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:12.237190 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:17.257075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:22.273842 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:27.293769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:32.327929 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:37.347719 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:42.369674 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:47.435675 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:52.453228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:03:57.743258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:02.765448 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:07.788312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:12.801835 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:17.824823 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:19.992222 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 7 items received I0617 21:04:22.842197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:27.884268 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:32.902556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:37.929890 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:42.946123 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:47.978185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:52.992851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:04:58.013906 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:03.059611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:08.088353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:13.115235 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:18.158297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:23.445723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:28.470211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:33.493247 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:38.516347 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:43.539928 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:48.567126 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:54.108485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:05:59.383771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:04.407927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:09.426520 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:14.456851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:19.478364 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:24.509973 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:29.531425 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:34.565196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:39.583562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:44.607757 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:49.626451 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:54.649556 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:06:59.673767 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:04.691672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:09.758294 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:10.201257 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 10 items received I0617 21:07:14.778548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:19.798798 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:24.825100 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:29.846490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:34.862005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:39.889962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:44.911355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:49.941021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:07:55.002716 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:00.039560 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:05.054551 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:10.079879 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:10.648115 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 4 items received I0617 21:08:15.096666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:20.247196 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:25.278864 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:30.309971 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:35.321831 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:40.341954 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:45.360418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:50.386897 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:08:55.407360 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:00.428919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:05.441965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:10.461851 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:15.479260 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:20.493434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:25.508885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:30.530709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:35.549836 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:40.611616 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:45.633450 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:50.657231 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:09:55.710224 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:00.729611 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:05.750943 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:10.771791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:15.789438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:20.808165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:22.522066 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 21:10:25.830120 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:30.855509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:35.877549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:40.898869 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:45.954287 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:50.973797 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:10:56.018494 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:01.038342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:06.061966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:11.080965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:16.101743 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:21.123001 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:26.153663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:31.189785 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:36.211331 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:41.227912 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:46.251834 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:51.275358 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:11:56.377861 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:01.408710 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:04.018814 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 21:12:06.426420 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:11.444059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:16.471860 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:21.488038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:26.515649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:31.533128 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:36.676046 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:41.694038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:46.711344 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:51.731330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:12:57.080888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:02.105803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:07.127267 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:12.145374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:17.164212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:22.181200 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:27.197818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:32.212931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:37.229984 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:42.255475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:47.275907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:52.293080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:13:57.376220 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:02.398181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:02.665167 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 21:14:07.415197 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:12.433772 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:17.451820 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:22.472384 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:27.580408 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:32.599549 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:37.615663 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:42.639326 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:47.659418 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:52.681017 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:14:57.696691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:02.714322 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:07.731485 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:12.747873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:17.783258 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:22.802276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:27.820511 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:32.838218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:37.853550 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:42.879736 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:44.218671 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 21:15:47.897308 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:52.914142 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:15:57.933753 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:02.952995 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:07.972020 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:12.989343 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:18.006597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:23.025966 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:28.040214 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:30.545556 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 21:16:33.059937 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:38.091893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:43.111066 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:48.125274 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:53.158244 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:16:58.176156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:03.195092 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:08.235711 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:13.260056 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:14.558287 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:17:14.558567 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 21:17:14.558636 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 21:17:14.558703 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:17:14.558736 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:17:14.558789 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.558819 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.558870 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:17:14.558908 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:17:14.558933 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:17:14.558995 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:17:14.559023 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:17:14.559041 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:17:14.559074 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:17:14.559146 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:17:14.559172 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:17:14.559189 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:17:14.559226 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 21:17:14.559271 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:17:14.559302 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:17:14.559345 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.559373 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.559404 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:17:14.559430 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:17:14.559477 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:17:14.559510 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:17:14.559558 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:17:14.559569 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:17:14.559575 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:17:14.559624 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:17:14.559654 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:17:14.559683 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:17:14.560091 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:17:14.560249 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 21:17:14.560267 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:17:14.560295 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 21:17:14.560312 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 21:17:14.560328 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 21:17:14.560337 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 21:17:14.560344 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 21:17:14.560367 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 21:17:14.560377 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 21:17:14.560403 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:17:14.560415 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 21:17:14.560427 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 21:17:14.560431 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 21:17:14.560436 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 21:17:14.560441 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 21:17:14.560447 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 21:17:14.560475 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:17:14.560511 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:17:14.560532 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.560549 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.560584 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:17:14.560617 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:17:14.560639 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:17:14.560690 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:17:14.560738 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:17:14.560762 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:17:14.560781 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:17:14.560862 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:17:14.560885 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:17:14.560908 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:17:14.560989 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 21:17:14.635656 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:17:14.635871 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:17:14.635938 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:17:14.636433 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:17:14.636489 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:17:14.636525 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:17:14.636592 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:17:14.636631 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:17:14.636662 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.636690 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:17:14.636746 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:17:14.636784 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:17:14.636817 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:17:14.636864 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:17:14.636895 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:17:14.636937 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:17:14.636118 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:17:14.636975 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:17:14.637050 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:17:14.637100 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:17:14.637130 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:17:14.711590 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:17:14.711671 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:17:14.711739 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:17:14.711796 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:17:14.711818 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:17:14.711842 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:17:14.712252 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:17:18.279679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:23.340098 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:28.377172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:33.401707 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:38.430597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:43.544016 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:48.561475 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:53.685145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:17:58.706132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:03.719514 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:08.751215 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:13.778202 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:18.797389 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:23.859543 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:28.881746 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:33.901629 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:38.919761 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:43.943645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:48.962894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:53.993059 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:18:59.022855 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:04.042355 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:09.062938 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:14.080900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:19.099429 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:24.136676 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:29.151013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:34.169377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:39.186036 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:44.206054 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:49.224826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:54.246353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:19:59.268239 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:04.283518 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:09.307140 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:13.680325 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 7 items received I0617 21:20:14.322226 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:19.337587 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:24.382133 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:25.034175 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 2 items received I0617 21:20:29.399361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:34.418885 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:39.438685 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:44.456749 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:49.479394 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:54.841021 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:20:59.860574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:04.880381 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:09.903249 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:14.923591 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:19.943600 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:24.973579 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:29.994965 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:35.013256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:40.028341 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:45.052924 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:50.072336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:21:55.127106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:00.145312 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:05.171992 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:10.186935 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:15.207143 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:18.235410 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 3 items received I0617 21:22:20.220639 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:25.438293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:30.454153 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:35.484588 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:40.503404 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:45.522733 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:50.534893 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:22:55.565345 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:00.605265 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:05.620064 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:10.634216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:15.648079 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:20.660666 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:25.675507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:30.687779 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:35.700862 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:40.720665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:45.740293 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:50.755748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:23:55.648755 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 21:23:55.776432 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:00.789351 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:05.884618 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:10.902546 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:15.965850 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:20.980187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:25.994398 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:31.009878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:36.026182 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:41.039181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:46.057968 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:51.078319 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:24:56.101362 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:01.122014 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:06.149899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:11.163318 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:16.191509 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:21.215959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:27.036074 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:32.055852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:37.073908 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:42.093605 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:47.109709 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:52.125920 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:25:57.146669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:02.176067 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:02.687571 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 21:26:07.195623 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:12.228903 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:17.247383 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:22.263109 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:27.286585 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:32.304434 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:37.321584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:42.341773 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:47.359217 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:52.382670 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:26:57.542962 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:02.608548 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:07.630734 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:12.081145 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 0 items received I0617 21:27:12.650694 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:17.670661 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:22.697270 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:27.712352 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:32.732003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:37.753103 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:42.772019 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:47.791723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:52.809949 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:27:57.830978 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:02.852478 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:07.868817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:12.882803 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:17.905682 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:22.924584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:27.947880 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:32.966156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:37.990723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:43.011127 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:48.036038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:53.050208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:28:58.069027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:03.084354 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:08.105687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:13.140804 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:18.162603 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:23.179700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:28.199857 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:33.222900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:38.243988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:43.341256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:48.359013 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:53.412479 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:29:58.431400 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:03.494374 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:08.512687 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:13.528208 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:15.290805 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 0 items received I0617 21:30:18.543288 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:23.777900 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:28.801111 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:33.821672 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:38.841941 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:43.870413 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:48.895518 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:53.921390 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:30:58.940516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:03.957778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:08.982397 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:14.003649 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:19.071307 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:24.096782 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:29.127010 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:34.143421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:39.168121 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:43.705592 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 21:31:44.189241 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:49.215311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:54.244009 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:31:59.266316 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:00.686182 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 7 items received I0617 21:32:04.284186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:09.313883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:14.330271 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:14.559261 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:32:14.559348 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 21:32:14.559364 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 21:32:14.559376 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:32:14.559385 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:32:14.559394 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.559399 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.559407 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:32:14.559416 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:32:14.559438 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:32:14.559442 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:32:14.559449 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:32:14.559481 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.559487 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.559493 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:32:14.559501 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:32:14.559519 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:32:14.559529 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:32:14.559561 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:32:14.559568 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:32:14.559449 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:32:14.559577 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:32:14.559635 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:32:14.559640 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:32:14.559641 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:32:14.559655 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:32:14.559660 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:32:14.559646 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:32:14.559679 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:32:14.559685 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:32:14.559691 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:32:14.559704 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 21:32:14.560196 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:32:14.560235 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:32:14.560251 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 21:32:14.560257 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:32:14.560258 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 21:32:14.560267 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 21:32:14.560267 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 21:32:14.560276 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 21:32:14.560276 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 21:32:14.560281 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 21:32:14.560283 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 21:32:14.560288 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 21:32:14.560288 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 21:32:14.560294 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 21:32:14.560322 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 21:32:14.560294 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 21:32:14.560339 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 21:32:14.560350 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:32:14.560355 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:32:14.560362 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.560367 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.560376 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:32:14.560427 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:32:14.560434 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:32:14.560444 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:32:14.560458 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:32:14.560464 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:32:14.560472 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:32:14.560525 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:32:14.560532 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:32:14.560539 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:32:14.560559 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 21:32:14.613516 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:32:14.613539 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:32:14.613593 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:32:14.613614 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:32:14.613621 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:32:14.613638 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:32:14.613713 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:32:14.613723 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:32:14.613731 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.613767 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:32:14.613780 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:32:14.613790 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:32:14.613799 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:32:14.613812 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:32:14.613849 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:32:14.613860 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:32:14.613870 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:32:14.613899 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:32:14.613947 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:32:14.613956 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:32:14.614411 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 21:32:14.693607 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:32:14.693650 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:32:14.693716 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:32:14.694242 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:32:14.694263 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:32:14.694333 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:32:14.694276 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:32:19.355624 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:24.383292 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:29.398385 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:34.421926 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:39.440769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:44.471677 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:49.490225 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:54.837800 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:32:59.854392 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:04.869240 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:09.884696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:14.900918 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:19.917575 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:24.947472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:29.968192 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:34.987559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:40.006018 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:45.023975 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:50.045049 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:33:55.151477 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:00.167361 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:05.187499 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:10.231491 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:15.253024 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:20.269037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:25.288172 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:30.331132 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:35.343636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:40.366576 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:45.386346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:50.406927 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:34:55.436724 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:00.453440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:05.468135 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:10.489330 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:15.507536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:20.522950 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:25.865459 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:30.881512 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:35.904174 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:40.925959 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:45.971536 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:50.986373 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:35:56.011007 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:01.039365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:06.067494 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:11.084218 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:16.104806 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:21.120045 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:26.140894 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:31.156679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:36.180357 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:41.197484 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:43.094499 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 9 items received I0617 21:36:46.226521 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:51.244895 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:36:56.266129 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:01.285080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:06.304086 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:11.322580 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:16.341369 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:21.455838 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:26.741191 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:31.772094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:36.789147 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:41.826486 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:46.844428 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:51.860075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:37:57.075005 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:02.094819 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:07.111165 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:12.130567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:17.153297 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:22.166696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:24.696927 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 3 items received I0617 21:38:26.356088 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 8 items received I0617 21:38:27.366573 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:32.383597 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:37.406259 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:42.425751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:47.441770 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:52.460480 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:38:57.484922 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:02.501777 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:07.525988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:12.547919 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:17.648276 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:22.668440 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:27.682256 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:32.696212 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:37.803027 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:42.833888 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:47.848311 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:52.865771 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:39:57.886592 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:02.908667 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:07.930930 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:12.949878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:17.967148 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:22.985934 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:28.013346 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:33.037028 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:38.058516 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:43.094778 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:48.112025 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:53.137303 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:40:58.156284 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:03.179031 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:08.197252 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:10.720769 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 6 items received I0617 21:41:13.212878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:18.236419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:23.373211 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:28.391607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:33.422566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:38.474562 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:43.495705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:48.525246 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:53.542584 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:41:58.568510 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:03.591507 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:08.622084 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:13.706883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:18.728457 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:23.872382 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:28.890070 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:33.903852 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:38.919705 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:43.941438 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:48.963421 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:54.059602 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:42:59.079528 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:04.098976 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:09.123492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:14.141826 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:19.156055 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:24.334206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:29.348544 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:34.426365 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:39.442636 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:44.459737 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:49.474455 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:54.490353 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:43:59.509768 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:04.529003 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:09.541594 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:14.565464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:19.584162 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:24.107779 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 6 items received I0617 21:44:24.973464 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:29.993106 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:35.008522 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:40.034309 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:45.050080 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:50.063907 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:44:55.164816 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:00.196883 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:05.211145 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:10.224970 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:15.245645 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:20.262527 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:25.280186 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:30.307405 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:35.325690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:40.344863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:45.361158 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:50.378751 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:45:54.713550 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 0 items received I0617 21:45:55.399723 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:00.420216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:05.434988 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:10.455228 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:15.479075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:20.499665 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:25.601185 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:26.726133 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotContent total 0 items received I0617 21:46:30.697195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:35.711011 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:40.724791 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:45.740073 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:50.752769 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:46:55.775399 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:00.786873 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:05.801181 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:10.821856 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:14.561482 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:47:14.561555 1 snapshot_controller_base.go:200] enqueued "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" for sync I0617 21:47:14.561571 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:47:14.561586 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978] I0617 21:47:14.561602 1 util.go:218] storeObjectUpdate updating content "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" with version 8127243 I0617 21:47:14.561612 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: content is bound to snapshot near/data-nearu-0-20220617014011 I0617 21:47:14.561617 1 snapshot_controller.go:90] syncContent[snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978]: check if we should add invalid label on content I0617 21:47:14.561624 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot near/data-nearu-0-20220617014011 found I0617 21:47:14.561632 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[near/data-nearu-0-20220617014011] I0617 21:47:14.561637 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [near/data-nearu-0-20220617014011]: update snapshot status to true if needed. I0617 21:47:14.561652 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:47:14.561658 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8110984 I0617 21:47:14.561665 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 21:47:14.561669 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 21:47:14.561674 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 21:47:14.561679 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 21:47:14.561682 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 21:47:14.561692 1 reflector.go:382] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: forcing resync I0617 21:47:14.561720 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:47:14.561724 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:47:14.561731 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.561737 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.561746 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:47:14.561754 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:47:14.561760 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:47:14.561767 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:47:14.561776 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:47:14.561780 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:47:14.561788 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:47:14.561838 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:47:14.561845 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:47:14.561849 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:47:14.561856 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found I0617 21:47:14.561864 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:47:14.561868 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:47:14.561872 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.561875 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.561878 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:47:14.561883 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:47:14.561886 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:47:14.561891 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:47:14.561897 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:47:14.561901 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:47:14.561906 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:47:14.561917 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:47:14.561927 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:47:14.561930 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:47:14.562318 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 21:47:14.562330 1 snapshot_controller_base.go:183] enqueued "near/data-nearu-0-20220617014011" for sync I0617 21:47:14.562336 1 snapshot_controller_base.go:227] syncSnapshotByKey[near/data-nearu-0-20220617014011] I0617 21:47:14.562339 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [near] name [data-nearu-0-20220617014011] I0617 21:47:14.562345 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-nearu-0-20220617014011]: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.562348 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.562362 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:47:14.562367 1 snapshot_controller_base.go:247] Updating snapshot "near/data-nearu-0-20220617014011" I0617 21:47:14.562371 1 snapshot_controller_base.go:383] updateSnapshot "near/data-nearu-0-20220617014011" I0617 21:47:14.562378 1 util.go:218] storeObjectUpdate updating snapshot "near/data-nearu-0-20220617014011" with version 8127250 I0617 21:47:14.562385 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[near/data-nearu-0-20220617014011]: bound to: "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978", Completed: true I0617 21:47:14.562392 1 snapshot_controller.go:185] syncSnapshot [near/data-nearu-0-20220617014011]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:47:14.562399 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-nearu-0-20220617014011]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005f8910), CreationTime:time.Date(2022, time.June, 17, 1, 40, 11, 0, time.Local), ReadyToUse:(*bool)(0xc0003ee337), RestoreSize:(*resource.Quantity)(0xc000488bc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:47:14.562424 1 snapshot_controller.go:194] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add invalid label on snapshot I0617 21:47:14.562429 1 snapshot_controller.go:210] syncSnapshot[near/data-nearu-0-20220617014011]: validate snapshot to make sure source has been correctly specified I0617 21:47:14.562433 1 snapshot_controller.go:219] syncSnapshot[near/data-nearu-0-20220617014011]: check if we should add finalizers on snapshot I0617 21:47:14.562440 1 snapshot_controller.go:404] syncReadySnapshot[near/data-nearu-0-20220617014011]: VolumeSnapshotContent "snapcontent-4612e854-a1b2-4ce2-a55c-23c2f7dc1978" found E0617 21:47:14.650478 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:47:14.650504 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:47:14.650561 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:47:14.650608 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:47:14.650695 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:47:14.650772 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:47:14.650915 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:47:14.650985 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:47:14.651020 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.651081 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:47:14.651116 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:47:14.651170 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:47:14.651215 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:47:14.651273 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8110989 I0617 21:47:14.651317 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:47:14.651385 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:47:14.651420 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc0005ba0c0), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005a2137), RestoreSize:(*resource.Quantity)(0xc000524140), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:47:14.651553 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:47:14.651727 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:47:14.651751 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:47:14.651759 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:47:14.694192 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:47:14.694216 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:47:14.694239 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:47:14.694489 1 snapshot_controller.go:210] syncSnapshot[geth/data-gethu-0-20220617005748]: validate snapshot to make sure source has been correctly specified I0617 21:47:14.694550 1 snapshot_controller.go:219] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add finalizers on snapshot I0617 21:47:14.694607 1 snapshot_controller.go:404] syncReadySnapshot[geth/data-gethu-0-20220617005748]: VolumeSnapshotContent "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" found I0617 21:47:14.695048 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8110989", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:47:15.837690 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:19.379245 1 reflector.go:536] k8s.io/client-go/informers/factory.go:134: Watch close - *v1.PersistentVolumeClaim total 7 items received I0617 21:47:20.850037 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:25.866388 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:30.881931 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:35.898748 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:40.916589 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:45.936313 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:50.950342 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:47:55.986195 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:01.001277 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:06.015812 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:11.029038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:16.055700 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:21.071559 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:26.097493 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:31.111914 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:36.127426 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:41.141765 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:46.162684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:51.214566 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:48:56.233818 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:01.262539 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:06.282607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:11.306957 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:16.327899 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:21.342574 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:26.367696 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:31.396300 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:36.414944 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:41.443336 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:46.459206 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:51.482094 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:49:56.525739 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:01.551350 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:06.566878 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:11.588038 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:16.608257 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:21.628430 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:26.671490 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:31.968678 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:36.991472 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:42.020178 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:47.035008 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:52.053207 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:50:57.326524 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:02.347987 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:07.366567 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:12.386595 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:17.403216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:17.857762 1 snapshot_controller_base.go:183] enqueued "geth/data-gethu-0-20220617005748" for sync I0617 21:51:17.857798 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:51:17.857804 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:51:17.857811 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:17.857817 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:17.857824 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:51:17.857835 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:51:17.857841 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:51:17.857852 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:51:17.857868 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:17.857880 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:51:17.857887 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:17.857936 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:17.857949 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:17.857954 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:51:17.935826 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:17.935851 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:17.935872 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:51:17.935887 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:17.935892 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:51:17.935898 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:17.935915 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:51:17.935945 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:17.935995 1 snapshot_controller.go:875] isVolumeBeingCreatedFromSnapshot: no volume is being created from snapshot geth/data-gethu-0-20220617005748 I0617 21:51:17.936027 1 snapshot_controller.go:313] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent[geth/data-gethu-0-20220617005748]: Set VolumeSnapshotBeingDeleted annotation on the content [snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:51:17.936065 1 snapshot_controller.go:1561] setAnnVolumeSnapshotBeingDeleted: set annotation [snapshot.storage.kubernetes.io/volumesnapshot-being-deleted] on content [snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]. I0617 21:51:17.938038 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:17.958532 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8553504 I0617 21:51:17.958698 1 snapshot_controller.go:1583] setAnnVolumeSnapshotBeingDeleted: volume snapshot content &{TypeMeta:{Kind: APIVersion:} ObjectMeta:{Name:snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c GenerateName: Namespace: SelfLink: UID:58ccd8ca-3329-4aed-a21a-5559cba218a3 ResourceVersion:8553504 Generation:1 CreationTimestamp:2022-06-17 00:57:48 +0000 UTC DeletionTimestamp: DeletionGracePeriodSeconds: Labels:map[] Annotations:map[snapshot.storage.kubernetes.io/volumesnapshot-being-deleted:yes] OwnerReferences:[] Finalizers:[snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection] ClusterName: ManagedFields:[{Manager:csi-snapshotter Operation:Update APIVersion:snapshot.storage.k8s.io/v1beta1 Time:2022-06-17 00:59:29 +0000 UTC FieldsType:FieldsV1 FieldsV1:{"f:status":{"f:readyToUse":{}}} Subresource:} {Manager:snapshot-controller Operation:Update APIVersion:snapshot.storage.k8s.io/v1 Time:2022-06-17 21:51:17 +0000 UTC FieldsType:FieldsV1 FieldsV1:{"f:metadata":{"f:annotations":{".":{},"f:snapshot.storage.kubernetes.io/volumesnapshot-being-deleted":{}},"f:finalizers":{".":{},"v:\"snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection\"":{}}},"f:spec":{".":{},"f:deletionPolicy":{},"f:driver":{},"f:source":{".":{},"f:volumeHandle":{}},"f:volumeSnapshotClassName":{},"f:volumeSnapshotRef":{".":{},"f:apiVersion":{},"f:kind":{},"f:name":{},"f:namespace":{},"f:resourceVersion":{},"f:uid":{}}}} Subresource:}]} Spec:{VolumeSnapshotRef:{Kind:VolumeSnapshot Namespace:geth Name:data-gethu-0-20220617005748 UID:8ebc315e-1690-408c-b322-e640d9f3d75c APIVersion:snapshot.storage.k8s.io/v1 ResourceVersion:8110035 FieldPath:} DeletionPolicy:Delete Driver:ebs.csi.aws.com VolumeSnapshotClassName:0xc000466a70 Source:{VolumeHandle:0xc000466a50 SnapshotHandle:}} Status:0xc0005448a0} I0617 21:51:17.958852 1 snapshot_controller.go:326] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: set DeletionTimeStamp on content [snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]. I0617 21:51:17.959702 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:51:17.959881 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:51:17.959931 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8553504 I0617 21:51:17.959987 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 21:51:17.960008 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 21:51:17.960070 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 21:51:17.960095 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 21:51:17.960129 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 21:51:17.975998 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:51:17.976033 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:51:17.976046 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8553506 I0617 21:51:17.976054 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 21:51:17.976059 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 21:51:17.976068 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 21:51:17.976074 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 21:51:17.976079 1 snapshot_controller.go:143] synchronizing VolumeSnapshotContent for snapshot [geth/data-gethu-0-20220617005748]: update snapshot status to true if needed. I0617 21:51:17.976150 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:51:17.977481 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:17.977872 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:17.978100 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:17.978121 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:51:18.027372 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.027484 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.027534 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.027571 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.027608 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:51:18.027638 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:51:18.027663 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:18.027682 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:18.027784 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:51:18.027887 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:51:18.027918 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:51:18.027945 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:51:18.027971 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:18.027996 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:51:18.028018 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:18.028052 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:18.028090 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:18.028111 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:51:18.034330 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 21:51:18.093254 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.093350 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.093438 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:51:18.093473 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:18.093513 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:51:18.093532 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:18.093563 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:51:18.093601 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:18.093633 1 snapshot_controller.go:875] isVolumeBeingCreatedFromSnapshot: no volume is being created from snapshot geth/data-gethu-0-20220617005748 I0617 21:51:18.093676 1 snapshot_controller.go:313] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent[geth/data-gethu-0-20220617005748]: Set VolumeSnapshotBeingDeleted annotation on the content [snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:51:18.093696 1 snapshot_controller.go:326] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: set DeletionTimeStamp on content [snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]. I0617 21:51:18.094007 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:18.106246 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:51:18.106342 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:18.106427 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:18.106531 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:18.106563 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:51:18.200084 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.200189 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.200243 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.200260 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.200764 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:18.434402 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:51:18.436530 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:51:18.436553 1 util.go:218] storeObjectUpdate updating content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" with version 8553514 I0617 21:51:18.436622 1 snapshot_controller.go:88] synchronizing VolumeSnapshotContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: content is bound to snapshot geth/data-gethu-0-20220617005748 I0617 21:51:18.436629 1 snapshot_controller.go:90] syncContent[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c]: check if we should add invalid label on content I0617 21:51:18.436638 1 snapshot_controller.go:1550] getSnapshotFromStore: snapshot geth/data-gethu-0-20220617005748 found I0617 21:51:18.436645 1 snapshot_controller.go:1081] needsUpdateSnapshotStatus[geth/data-gethu-0-20220617005748] I0617 21:51:18.449024 1 snapshot_controller_base.go:200] enqueued "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" for sync I0617 21:51:18.449049 1 snapshot_controller_base.go:302] syncContentByKey[snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c] I0617 21:51:18.449065 1 snapshot_controller_base.go:463] content "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c" deleted I0617 21:51:18.449071 1 snapshot_controller_base.go:473] deleteContent["snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c"]: scheduling sync of snapshot geth/data-gethu-0-20220617005748 I0617 21:51:18.449086 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:51:18.449089 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:51:18.449095 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:18.449099 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:18.449104 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:51:18.449111 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:51:18.449115 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:51:18.449128 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:51:18.449138 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:18.449142 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:51:18.449151 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:18.449188 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:18.449199 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:18.449284 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:51:18.509540 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.509567 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.509588 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:51:18.509606 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:18.509611 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:51:18.509617 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:18.509632 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:51:18.509639 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:18.509648 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:51:18.509655 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:18.509677 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:18.509694 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:18.509700 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:51:18.510291 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 21:51:18.563654 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.563680 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:18.563705 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.563723 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:18.564396 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:19.029517 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:51:19.106773 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:51:19.107025 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:19.107127 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:19.107207 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:51:19.107290 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:51:19.107364 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:51:19.107442 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:51:19.107519 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:19.107605 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:51:19.107680 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:19.107821 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:19.107902 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:19.107970 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:51:19.193946 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:19.194465 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:19.194638 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:51:19.194708 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:19.194720 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:51:19.194725 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:19.194796 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:51:19.194811 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:19.194876 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:51:19.194933 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:19.195011 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:19.195078 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:19.195088 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:51:19.195319 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 21:51:19.312641 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:19.312929 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:19.313108 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:19.313697 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:19.313256 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:22.426684 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:27.326073 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:51:27.326134 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:51:27.326145 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:27.326152 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:27.326161 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:51:27.326353 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:51:27.326456 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:51:27.326543 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:51:27.326626 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:27.326707 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:51:27.326782 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:27.327010 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:27.327103 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:27.327181 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:51:27.515607 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:27.515933 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:27.516115 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:51:27.516463 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:27.516631 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:51:27.516719 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:27.516812 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:51:27.516957 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:27.517040 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:51:27.517187 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:27.517297 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:27.517382 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:27.516266 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:27.517708 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:51:27.534419 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader E0617 21:51:27.567314 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:27.567352 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:27.567404 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:27.567553 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:27.567759 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:32.553963 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:37.571062 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:42.585691 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:43.568491 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:51:43.568519 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:51:43.568573 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:43.568582 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:43.568601 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:51:43.568611 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:51:43.568619 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:51:43.568632 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:51:43.568642 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:43.568646 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:51:43.568653 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:43.568701 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:43.568712 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:43.568717 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:51:43.796829 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:43.801076 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:43.801138 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:51:43.801151 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:43.801157 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:51:43.801162 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:51:43.801176 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:51:43.801197 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:51:43.801214 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:51:43.801223 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:51:43.801251 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:51:43.801301 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:51:43.801308 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:51:43.802205 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 21:51:43.874182 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:43.874211 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:51:43.874257 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:43.874276 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:51:43.878114 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:51:47.606377 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:52.648896 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:51:57.680492 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:02.699409 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:07.719614 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:12.735463 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:15.925626 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:52:15.925684 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:52:15.925696 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:52:15.925702 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:52:15.925710 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:52:15.925734 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:52:15.925742 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:52:15.925757 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:52:15.925771 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:52:15.925808 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:52:15.925816 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:52:15.925842 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:52:15.925858 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:52:15.925863 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:52:16.037151 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:52:16.037257 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:52:16.037370 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:52:16.037584 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:52:16.037594 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:52:16.037599 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:52:16.037618 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:52:16.038433 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:52:16.038485 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:52:16.038576 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:52:16.038801 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:52:16.038840 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:52:16.038859 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation I0617 21:52:16.038348 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot E0617 21:52:16.085802 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:52:16.085892 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:52:16.086027 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:52:16.086102 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:52:16.086695 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:52:17.760041 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:22.775187 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:27.813216 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:32.827470 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:37.854075 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:42.869248 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:47.890669 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:52.908542 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:52:57.930679 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:02.955108 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:05.112780 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshotClass total 4 items received I0617 21:53:06.718111 1 reflector.go:536] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Watch close - *v1.VolumeSnapshot total 8 items received I0617 21:53:07.987301 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:13.005607 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:18.026427 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:20.086673 1 snapshot_controller_base.go:227] syncSnapshotByKey[geth/data-gethu-0-20220617005748] I0617 21:53:20.086836 1 snapshot_controller_base.go:230] snapshotWorker: snapshot namespace [geth] name [data-gethu-0-20220617005748] I0617 21:53:20.087017 1 snapshot_controller_base.go:353] checkAndUpdateSnapshotClass [data-gethu-0-20220617005748]: VolumeSnapshotClassName [aws-block-storage] I0617 21:53:20.087113 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:53:20.087223 1 snapshot_controller_base.go:373] VolumeSnapshotClass [aws-block-storage] Driver [ebs.csi.aws.com] I0617 21:53:20.087316 1 snapshot_controller_base.go:247] Updating snapshot "geth/data-gethu-0-20220617005748" I0617 21:53:20.087404 1 snapshot_controller_base.go:383] updateSnapshot "geth/data-gethu-0-20220617005748" I0617 21:53:20.087531 1 util.go:218] storeObjectUpdate updating snapshot "geth/data-gethu-0-20220617005748" with version 8553502 I0617 21:53:20.087652 1 snapshot_controller.go:183] synchronizing VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:53:20.087741 1 snapshot_controller.go:185] syncSnapshot [geth/data-gethu-0-20220617005748]: check if we should remove finalizer on snapshot PVC source and remove it if we can I0617 21:53:20.087846 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:53:20.087995 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:53:20.088100 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:53:20.088186 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:53:20.215446 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:53:20.215760 1 snapshot_controller.go:189] error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:53:20.215997 1 snapshot_controller.go:194] syncSnapshot[geth/data-gethu-0-20220617005748]: check if we should add invalid label on snapshot I0617 21:53:20.216506 1 snapshot_controller.go:239] processSnapshotWithDeletionTimestamp VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:53:20.216824 1 snapshot_controller.go:1298] getSnapshotDriverName: VolumeSnapshot[data-gethu-0-20220617005748] I0617 21:53:20.216923 1 snapshot_controller.go:1284] getSnapshotClass: VolumeSnapshotClassName [aws-block-storage] I0617 21:53:20.217013 1 snapshot_controller.go:287] processSnapshotWithDeletionTimestamp[geth/data-gethu-0-20220617005748]: delete snapshot content and remove finalizer from snapshot if needed I0617 21:53:20.216184 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:53:20.217548 1 snapshot_controller.go:294] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[geth/data-gethu-0-20220617005748]: bound to: "snapcontent-8ebc315e-1690-408c-b322-e640d9f3d75c", Completed: true I0617 21:53:20.217686 1 snapshot_controller.go:334] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[geth/data-gethu-0-20220617005748] I0617 21:53:20.217762 1 snapshot_controller.go:982] checkandRemovePVCFinalizer for snapshot [data-gethu-0-20220617005748]: snapshot status [&v1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc00053dc00), CreationTime:time.Date(2022, time.June, 17, 0, 57, 48, 0, time.Local), ReadyToUse:(*bool)(0xc0005358b7), RestoreSize:(*resource.Quantity)(0xc0005e3fc0), Error:(*v1.VolumeSnapshotError)(nil)}] I0617 21:53:20.217980 1 snapshot_controller.go:937] Checking isPVCBeingUsed for snapshot [geth/data-gethu-0-20220617005748] I0617 21:53:20.218088 1 snapshot_controller.go:962] isPVCBeingUsed: no snapshot is being created from PVC geth/data-gethu-0 I0617 21:53:20.218159 1 snapshot_controller.go:990] checkandRemovePVCFinalizer[data-gethu-0-20220617005748]: Remove Finalizer for PVC data-gethu-0 as it is not used by snapshots in creation E0617 21:53:20.273638 1 snapshot_controller.go:993] checkandRemovePVCFinalizer [data-gethu-0-20220617005748]: removePVCFinalizer failed to remove finalizer snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:53:20.273807 1 snapshot_controller.go:1503] removeSnapshotFinalizer: error check and remove PVC finalizer for snapshot [data-gethu-0-20220617005748]: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } E0617 21:53:20.273912 1 snapshot_controller_base.go:399] could not sync snapshot "geth/data-gethu-0-20220617005748": snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:53:20.274049 1 snapshot_controller_base.go:217] Failed to sync snapshot "geth/data-gethu-0-20220617005748", will retry again: snapshot controller failed to update data-gethu-0-20220617005748 on API server: snapshot controller failed to update data-gethu-0 on API server: PersistentVolumeClaim "data-gethu-0" is invalid: spec: Forbidden: spec is immutable after creation except resources.requests for bound claims   core.PersistentVolumeClaimSpec{    ... // 4 identical fields    StorageClassName: &"ebs-sc",    VolumeMode: &"Filesystem",    DataSource: &core.TypedLocalObjectReference{    APIGroup: &"snapshot.storage.k8s.io",    Kind: "VolumeSnapshot", -  Name: "data-gethu-0-20220617005748", +  Name: "data-gethu-0-20220609235015",    },    DataSourceRef: nil,   } I0617 21:53:20.274195 1 event.go:285] Event(v1.ObjectReference{Kind:"VolumeSnapshot", Namespace:"geth", Name:"data-gethu-0-20220617005748", UID:"8ebc315e-1690-408c-b322-e640d9f3d75c", APIVersion:"snapshot.storage.k8s.io/v1", ResourceVersion:"8553502", FieldPath:""}): type: 'Warning' reason: 'ErrorPVCFinalizer' Error check and remove PVC Finalizer for VolumeSnapshot I0617 21:53:23.044863 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:28.094156 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:33.122332 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:38.143050 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:43.177817 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader I0617 21:53:48.203022 1 leaderelection.go:278] successfully renewed lease kube-system/snapshot-controller-leader