root@deploy2002:~# kube_env admin dse-k8s-codfw
root@deploy2002:~# kubectl create namespace stevemunene-pvc-tests
namespace/stevemunene-pvc-tests created
root@deploy2002:~# kubectl get namespaces
NAME                    STATUS   AGE
analytics-test          Active   47h
cert-manager            Active   10d
default                 Active   10d
echoserver              Active   10d
external-services       Active   10d
istio-system            Active   10d
kube-node-lease         Active   10d
kube-public             Active   10d
kube-system             Active   10d
opensearch-ipoid        Active   10d
opensearch-ipoid-test   Active   10d
opensearch-operator     Active   10d
opensearch-test         Active   10d
sidecar-controller      Active   10d
stevemunene-pvc-tests   Active   13s


root@deploy2002:~# cat /home/stevemunene/raw-block-pvc.yaml 
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: raw-block-pvc
  namespace: stevemunene-pvc-tests
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Block
  resources:
    requests:
      storage: 1Gi
  storageClassName: ceph-rbd-ssd

---
apiVersion: v1
kind: Pod
metadata:
  name: pod-with-raw-block-volume
  namespace: stevemunene-pvc-tests
spec:
  containers:
    - name: do-nothing
      image: docker-registry.discovery.wmnet/bookworm:20240630
      command: ["/bin/sh", "-c"]
      args: ["tail -f /dev/null"]
      volumeDevices:
        - name: data
          devicePath: /dev/xvda
  volumes:
    - name: data
      persistentVolumeClaim:
        claimName: raw-block-pvc

root@deploy2002:~# kubectl -n stevemunene-pvc-tests get events -w
LAST SEEN   TYPE      REASON                 OBJECT                                MESSAGE
5m32s       Warning   FailedScheduling       pod/pod-with-raw-block-volume         0/4 nodes are available: pod has unbound immediate PersistentVolumeClaims. preemption: 0/4 nodes are available: 4 Preemption is not helpful for scheduling.
2m40s       Normal    Provisioning           persistentvolumeclaim/raw-block-pvc   External provisioner is provisioning volume for claim "stevemunene-pvc-tests/raw-block-pvc"
71s         Normal    ExternalProvisioning   persistentvolumeclaim/raw-block-pvc   Waiting for a volume to be created either by the external provisioner 'rbd.csi.ceph.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered.
0s          Normal    Provisioning           persistentvolumeclaim/raw-block-pvc   External provisioner is provisioning volume for claim "stevemunene-pvc-tests/raw-block-pvc"
0s          Normal    ExternalProvisioning   persistentvolumeclaim/raw-block-pvc   Waiting for a volume to be created either by the external provisioner 'rbd.csi.ceph.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered.
0s          Normal    Provisioning           persistentvolumeclaim/raw-block-pvc   External provisioner is provisioning volume for claim "stevemunene-pvc-tests/raw-block-pvc"

^Croot@deploy2002:~# kubectl -n kube-system get events -w
LAST SEEN   TYPE      REASON             OBJECT                                            MESSAGE
11m         Warning   FailedScheduling   pod/calico-typha-7c5b67b766-g4fbr                 0/4 nodes are available: 2 node(s) didn't have free ports for the requested pod ports, 2 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }. preemption: 0/4 nodes are available: 2 Preemption is not helpful for scheduling, 2 node(s) didn't have free ports for the requested pod ports.
11m         Warning   FailedScheduling   pod/ceph-csi-cephfs-provisioner-5f8874f66-hdd8j   0/4 nodes are available: 2 node(s) didn't match pod anti-affinity rules, 2 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }. preemption: 0/4 nodes are available: 2 Preemption is not helpful for scheduling, 2 node(s) didn't match pod anti-affinity rules.
11m

root@deploy2002:~# kubectl get pod pod-with-raw-block-volume -o yaml -n stevemunene-pvc-tests 
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{},"name":"pod-with-raw-block-volume","namespace":"stevemunene-pvc-tests"},"spec":{"containers":[{"args":["tail -f /dev/null"],"command":["/bin/sh","-c"],"image":"docker-registry.discovery.wmnet/bookworm:20240630","name":"do-nothing","volumeDevices":[{"devicePath":"/dev/xvda","name":"data"}]}],"volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"raw-block-pvc"}}]}}
  creationTimestamp: "2025-10-07T06:51:06Z"
  name: pod-with-raw-block-volume
  namespace: stevemunene-pvc-tests
  resourceVersion: "10912722"
  uid: c6137e26-ff1d-4706-a12a-fee08ce8f45d
spec:
  containers:
  - args:
    - tail -f /dev/null
    command:
    - /bin/sh
    - -c
    image: docker-registry.discovery.wmnet/bookworm:20240630
    imagePullPolicy: IfNotPresent
    name: do-nothing
    resources: {}
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeDevices:
    - devicePath: /dev/xvda
      name: data
    volumeMounts:
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-vhrxd
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext: {}
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: raw-block-pvc
  - name: kube-api-access-vhrxd
    projected:
      defaultMode: 420
      sources:
      - serviceAccountToken:
          expirationSeconds: 3607
          path: token
      - configMap:
          items:
          - key: ca.crt
            path: ca.crt
          name: kube-root-ca.crt
      - downwardAPI:
          items:
          - fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
            path: namespace
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2025-10-07T06:51:06Z"
    message: '0/4 nodes are available: pod has unbound immediate PersistentVolumeClaims.
      preemption: 0/4 nodes are available: 4 Preemption is not helpful for scheduling.'
    reason: Unschedulable
    status: "False"
    type: PodScheduled
  phase: Pending
  qosClass: BestEffort

root@deploy2002:~# kubectl get pvc -n stevemunene-pvc-tests 
NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
raw-block-pvc   Pending                                      ceph-rbd-ssd   <unset>                 3d22h

root@deploy2002:~# kubectl get pvc -n stevemunene-pvc-tests 
NAME            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
raw-block-pvc   Pending                                      ceph-rbd-ssd   <unset>                 3d22h
root@deploy2002:~# kubectl describe pvc raw-block-pvc -n stevemunene-pvc-tests 
Name:          raw-block-pvc
Namespace:     stevemunene-pvc-tests
StorageClass:  ceph-rbd-ssd
Status:        Pending
Volume:        
Labels:        <none>
Annotations:   volume.beta.kubernetes.io/storage-provisioner: rbd.csi.ceph.com
               volume.kubernetes.io/storage-provisioner: rbd.csi.ceph.com
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      
Access Modes:  
VolumeMode:    Block
Used By:       pod-with-raw-block-volume
Events:
  Type    Reason                Age                        From                                                                                             Message
  ----    ------                ----                       ----                                                                                             -------
  Normal  ExternalProvisioning  3m50s (x22605 over 3d22h)  persistentvolume-controller                                                                      Waiting for a volume to be created either by the external provisioner 'rbd.csi.ceph.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered.
  Normal  Provisioning          19s (x1517 over 3d22h)     rbd.csi.ceph.com_ceph-csi-rbd-provisioner-69c74d89cc-znf98_e2ab8de1-a73b-47e5-82fe-7f89b39d87fd  External provisioner is provisioning volume for claim "stevemunene-pvc-tests/raw-block-pvc"

root@deploy2002:~# kubectl get csidriver 
NAME                  ATTACHREQUIRED   PODINFOONMOUNT   STORAGECAPACITY   TOKENREQUESTS   REQUIRESREPUBLISH   MODES        AGE
cephfs.csi.ceph.com   false            false            false             <unset>         false               Persistent   17d
rbd.csi.ceph.com      true             false            false             <unset>         false               Persistent   17d

root@deploy2002:~# kubectl delete pvc raw-block-pvc -n stevemunene-pvc-tests 
persistentvolumeclaim "raw-block-pvc" deleted
root@deploy2002:~# kubectl get pods -n stevemunene-pvc-tests 
No resources found in stevemunene-pvc-tests namespace.
root@deploy2002:~# kubectl get pvc -n stevemunene-pvc-tests 
No resources found in stevemunene-pvc-tests namespace.
root@deploy2002:~#

0s          Normal    Provisioning           persistentvolumeclaim/raw-block-pvc   External provisioner is provisioning volume for claim "stevemunene-pvc-tests/raw-block-pvc"
0s          Warning   ProvisioningFailed     persistentvolumeclaim/raw-block-pvc   failed to provision volume with StorageClass "ceph-rbd-ssd": rpc error: code = Internal desc = rados: ret=-1, Operation not permitted
0s          Normal    Provisioning           persistentvolumeclaim/raw-block-pvc   External provisioner is provisioning volume for claim "stevemunene-pvc-tests/raw-block-pvc"
0s          Warning   ProvisioningFailed     persistentvolumeclaim/raw-block-pvc   failed to provision volume with StorageClass "ceph-rbd-ssd": rpc error: code = Internal desc = rados: ret=-1, Operation not permitted