Common labels: {"app":"ceph-csi-rbd","component":"nodeplugin","container":"csi-rbdplugin","filename":"/var/log/pods/ceph-csi-rbd_ceph-csi-rbd-nodeplugin-2f9gf_9cbee498-ec20-49ac-b9b0-f03ec831d34d/csi-rbdplugin/0.log","job":"ceph-csi-rbd/ceph-csi-rbd","namespace":"ceph-csi-rbd","node_name":"fiestel-node-s-jgedbg-0","pod":"ceph-csi-rbd-nodeplugin-2f9gf","stream":"stderr"} Line limit: "1000 (35 displayed)" Total bytes processed: "1.95 MB" 2024-11-17 05:10:07.309 I1117 04:10:07.309250 2166526 utils.go:247] ID: 296 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC response: {} 2024-11-17 05:10:07.309 I1117 04:10:07.309002 2166526 nodeserver.go:942] ID: 296 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 rbd: successfully unbound volume volume-490a870a-23fb-4662-b26b-ee5ad5841157 from /var/lib/kubelet/pods/2a8b0d55-0672-47c6-8085-5fad156d8376/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:10:07.248 I1117 04:10:07.248352 2166526 utils.go:241] ID: 296 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC request: {"target_path":"/var/lib/kubelet/pods/2a8b0d55-0672-47c6-8085-5fad156d8376/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount","volume_id":"volume-490a870a-23fb-4662-b26b-ee5ad5841157"} 2024-11-17 05:10:07.248 I1117 04:10:07.248185 2166526 utils.go:240] ID: 296 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC call: /csi.v1.Node/NodeUnpublishVolume 2024-11-17 05:10:00.654 I1117 04:10:00.654866 2166526 utils.go:247] ID: 282 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC response: {} 2024-11-17 05:10:00.654 I1117 04:10:00.654696 2166526 nodeserver.go:742] ID: 282 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 rbd: successfully mounted stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 to targetPath /var/lib/kubelet/pods/2a8b0d55-0672-47c6-8085-5fad156d8376/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:10:00.653 I1117 04:10:00.653112 2166526 mount_linux.go:218] Mounting cmd (mount) with arguments (-t xfs -o bind,remount,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 /var/lib/kubelet/pods/2a8b0d55-0672-47c6-8085-5fad156d8376/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount) 2024-11-17 05:10:00.637 I1117 04:10:00.637650 2166526 mount_linux.go:218] Mounting cmd (mount) with arguments (-t xfs -o bind,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 /var/lib/kubelet/pods/2a8b0d55-0672-47c6-8085-5fad156d8376/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount) 2024-11-17 05:10:00.637 stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 2024-11-17 05:10:00.637 I1117 04:10:00.637639 2166526 nodeserver.go:856] ID: 282 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 target /var/lib/kubelet/pods/2a8b0d55-0672-47c6-8085-5fad156d8376/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:10:00.637 I1117 04:10:00.637527 2166526 utils.go:241] ID: 282 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount","target_path":"/var/lib/kubelet/pods/2a8b0d55-0672-47c6-8085-5fad156d8376/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount","volume_capability":{"AccessType":{"Mount":{"fs_type":"xfs"}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"m-cloud","dataPool":"volumes_data","imageFeatures":"layering","imageName":"volume-490a870a-23fb-4662-b26b-ee5ad5841157","pool":"volumes","staticVolume":"true"},"volume_id":"volume-490a870a-23fb-4662-b26b-ee5ad5841157"} 2024-11-17 05:10:00.637 I1117 04:10:00.637477 2166526 utils.go:240] ID: 282 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC call: /csi.v1.Node/NodePublishVolume 2024-11-17 05:08:09.847 I1117 04:08:09.847680 2166526 utils.go:247] ID: 258 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC response: {} 2024-11-17 05:08:09.847 I1117 04:08:09.847483 2166526 nodeserver.go:942] ID: 258 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 rbd: successfully unbound volume volume-490a870a-23fb-4662-b26b-ee5ad5841157 from /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:08:09.006 I1117 04:08:09.006495 2166526 utils.go:241] ID: 258 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC request: {"target_path":"/var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount","volume_id":"volume-490a870a-23fb-4662-b26b-ee5ad5841157"} 2024-11-17 05:08:09.006 I1117 04:08:09.006450 2166526 utils.go:240] ID: 258 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC call: /csi.v1.Node/NodeUnpublishVolume 2024-11-17 05:08:08.469 E1117 04:08:08.469199 2166526 utils.go:245] ID: 254 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC error: rpc error: code = Internal desc = unlinkat /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount: device or resource busy 2024-11-17 05:08:05.647 I1117 04:08:05.647666 2166526 utils.go:241] ID: 254 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC request: {"target_path":"/var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount","volume_id":"volume-490a870a-23fb-4662-b26b-ee5ad5841157"} 2024-11-17 05:08:05.647 I1117 04:08:05.647346 2166526 utils.go:240] ID: 254 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC call: /csi.v1.Node/NodeUnpublishVolume 2024-11-17 05:07:41.848 I1117 04:07:41.848967 2166526 utils.go:247] ID: 244 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC response: {} 2024-11-17 05:07:41.848 I1117 04:07:41.848847 2166526 nodeserver.go:742] ID: 244 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 rbd: successfully mounted stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 to targetPath /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:07:41.847 I1117 04:07:41.847049 2166526 mount_linux.go:218] Mounting cmd (mount) with arguments (-t xfs -o bind,remount,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount) 2024-11-17 05:07:17.467 I1117 04:07:17.467123 2166526 utils.go:247] ID: 223 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC response: {} 2024-11-17 05:07:17.467 I1117 04:07:17.467016 2166526 nodeserver.go:742] ID: 223 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 rbd: successfully mounted stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 to targetPath /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:07:17.464 I1117 04:07:17.464585 2166526 mount_linux.go:218] Mounting cmd (mount) with arguments (-t xfs -o bind,remount,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount) 2024-11-17 05:07:02.368 I1117 04:07:02.368470 2166526 mount_linux.go:218] Mounting cmd (mount) with arguments (-t xfs -o bind,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount) 2024-11-17 05:07:02.368 stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 2024-11-17 05:07:02.368 I1117 04:07:02.368457 2166526 nodeserver.go:856] ID: 244 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 target /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:07:02.368 I1117 04:07:02.368415 2166526 utils.go:241] ID: 244 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount","target_path":"/var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount","volume_capability":{"AccessType":{"Mount":{"fs_type":"xfs"}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"m-cloud","dataPool":"volumes_data","imageFeatures":"layering","imageName":"volume-490a870a-23fb-4662-b26b-ee5ad5841157","pool":"volumes","staticVolume":"true"},"volume_id":"volume-490a870a-23fb-4662-b26b-ee5ad5841157"} 2024-11-17 05:07:02.368 I1117 04:07:02.368342 2166526 utils.go:240] ID: 244 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC call: /csi.v1.Node/NodePublishVolume 2024-11-17 05:05:01.675 I1117 04:05:01.675926 2166526 mount_linux.go:218] Mounting cmd (mount) with arguments (-t xfs -o bind,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount) 2024-11-17 05:05:01.675 stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount/volume-490a870a-23fb-4662-b26b-ee5ad5841157 2024-11-17 05:05:01.675 I1117 04:05:01.675839 2166526 nodeserver.go:856] ID: 223 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 target /var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount 2024-11-17 05:05:01.675 I1117 04:05:01.675444 2166526 utils.go:241] ID: 223 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/84f33d2a2b138df681a34b503f5010392608dbba027225913b747d47ad3d2d45/globalmount","target_path":"/var/lib/kubelet/pods/f148e3a7-ea39-48eb-89cb-b94039cdc288/volumes/kubernetes.io~csi/pv-shoot--chp-prod--fiestel-6d1ab771-e264-48e2-88fd-689de72642f9/mount","volume_capability":{"AccessType":{"Mount":{"fs_type":"xfs"}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"m-cloud","dataPool":"volumes_data","imageFeatures":"layering","imageName":"volume-490a870a-23fb-4662-b26b-ee5ad5841157","pool":"volumes","staticVolume":"true"},"volume_id":"volume-490a870a-23fb-4662-b26b-ee5ad5841157"} 2024-11-17 05:05:01.675 I1117 04:05:01.675202 2166526 utils.go:240] ID: 223 Req-ID: volume-490a870a-23fb-4662-b26b-ee5ad5841157 GRPC call: /csi.v1.Node/NodePublishVolume