Page MenuHomePhabricator
Files F34945326

kubernetes1014.log
JMeybohm
Actions

Authored By
JMeybohm
Feb 8 2022, 9:59 AM
Size
14 KB
Referenced Files
None
Subscribers
None

kubernetes1014.log
View Options

root@deploy1002:~# kubectl describe node kubernetes1014.eqiad.wmnet
Name: kubernetes1014.eqiad.wmnet
Roles: <none>
Labels: beta.kubernetes.io/arch=amd64
beta.kubernetes.io/os=linux
failure-domain.beta.kubernetes.io/region=eqiad
failure-domain.beta.kubernetes.io/zone=row-d
kubernetes.io/arch=amd64
kubernetes.io/hostname=kubernetes1014.eqiad.wmnet
kubernetes.io/os=linux
node.kubernetes.io/disk-type=ssd
Annotations: node.alpha.kubernetes.io/ttl: 0
projectcalico.org/IPv4Address: 10.64.48.230/22
projectcalico.org/IPv6Address: 2620:0:861:107:10:64:48:230/64
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Tue, 23 Mar 2021 09:53:07 +0000
Taints: <none>
Unschedulable: false
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
NetworkUnavailable False Thu, 09 Dec 2021 14:34:47 +0000 Thu, 09 Dec 2021 14:34:47 +0000 CalicoIsUp Calico is running on this
node
MemoryPressure False Mon, 07 Feb 2022 06:07:23 +0000 Tue, 23 Mar 2021 09:53:07 +0000 KubeletHasSufficientMemory kubelet has sufficient mem
ory available
DiskPressure False Mon, 07 Feb 2022 06:07:23 +0000 Wed, 29 Sep 2021 07:28:27 +0000 KubeletHasNoDiskPressure kubelet has no disk pressu
re
PIDPressure False Mon, 07 Feb 2022 06:07:23 +0000 Tue, 23 Mar 2021 09:53:07 +0000 KubeletHasSufficientPID kubelet has sufficient PID
available
Ready True Mon, 07 Feb 2022 06:07:23 +0000 Wed, 29 Sep 2021 09:46:08 +0000 KubeletReady kubelet is posting ready s
tatus. AppArmor enabled
Addresses:
InternalIP: 10.64.48.230
Hostname: kubernetes1014.eqiad.wmnet
Capacity:
cpu: 48
ephemeral-storage: 28688292Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 97409220Ki
pods: 110
Allocatable:
cpu: 48
ephemeral-storage: 26439129864
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 97306820Ki
pods: 110
System Info:
Machine ID: 73679900aa984a45bf760f538524aba6
System UUID: 4c4c4544-004b-4310-8051-b1c04f473133
Boot ID: 7d518087-e900-48d4-86bd-3ce1104ad812
Kernel Version: 4.19.0-0.bpo.14-amd64
OS Image: Debian GNU/Linux 9 (stretch)
Operating System: linux
Architecture: amd64
Container Runtime Version: docker://18.6.3
Kubelet Version: v1.16.15
Kube-Proxy Version: v1.16.15
Non-terminated Pods: (35 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE
--------- ---- ------------ ---------- --------------- ------------- ---
apertium apertium-production-6b99ff7694-8d6pw 300m (0%) 1600m (3%) 300Mi (0%) 1300Mi (1%) 20d
api-gateway api-gateway-production-5889589fc-dz2p8 900m (1%) 1400m (2%) 900Mi (0%) 1200Mi (1%) 12d
blubberoid blubberoid-production-66947dddf9-4df7f 300m (0%) 1500m (3%) 200Mi (0%) 600Mi (0%) 5d17
h
changeprop-jobqueue changeprop-production-b5fbcffbb-x9q7n 700m (1%) 1200m (2%) 500Mi (0%) 1700Mi (1%) 18d
changeprop changeprop-production-78fb8b7cd4-hfpwf 700m (1%) 3200m (6%) 500Mi (0%) 1700Mi (1%) 18d
cxserver cxserver-production-694c7db9c-6zkjh 300m (0%) 1500m (3%) 200Mi (0%) 900Mi (0%) 3d18
h
eventgate-analytics-external eventgate-analytics-external-production-7577b69644-lzzhx 400m (0%) 2500m (5%) 250Mi (0%) 800Mi (0%) 11d
eventgate-analytics eventgate-analytics-production-7b8ccf799b-gwhzj 400m (0%) 2500m (5%) 250Mi (0%) 800Mi (0%) 10d
eventgate-analytics eventgate-analytics-production-7b8ccf799b-twbq8 400m (0%) 2500m (5%) 250Mi (0%) 800Mi (0%) 10d
eventgate-logging-external eventgate-logging-external-production-99fd8cc75-m2l47 700m (1%) 3 (6%) 250Mi (0%) 800Mi (0%) 11d
eventgate-main eventgate-main-production-fb5d896b9-rdxsn 400m (0%) 2500m (5%) 250Mi (0%) 1100Mi (1%) 4d16
h
eventstreams-internal eventstreams-internal-main-5bffd6d7c6-t2xkt 400m (0%) 2 (4%) 600Mi (0%) 1500Mi (1%) 10d
eventstreams eventstreams-production-7b64d578d8-nzhzq 400m (0%) 2 (4%) 600Mi (0%) 1500Mi (1%) 10d
kube-system calico-node-86v2g 350m (0%) 0 (0%) 400Mi (0%) 400Mi (0%) 59d
kube-system coredns-584d8549b5-hr6tg 100m (0%) 0 (0%) 70Mi (0%) 170Mi (0%) 130d
linkrecommendation linkrecommendation-internal-58fd7dd4d8-vvdh5 2050m (4%) 4600m (9%) 700Mi (0%) 1250Mi (1%) 3d17
h
mathoid mathoid-production-596cb7c944-v6brp 300m (0%) 1500m (3%) 300Mi (0%) 1524288k (1%) 73d
mathoid mathoid-production-596cb7c944-wmhrs 300m (0%) 1500m (3%) 300Mi (0%) 1524288k (1%) 73d
mobileapps mobileapps-production-746f7579c9-58b7q 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d
mobileapps mobileapps-production-746f7579c9-79hdk 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d
mobileapps mobileapps-production-746f7579c9-7wx27 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d
mobileapps mobileapps-production-746f7579c9-8ncx6 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d
mobileapps mobileapps-production-746f7579c9-96xk7 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d
mobileapps mobileapps-production-746f7579c9-ck7bl 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d
mobileapps mobileapps-production-746f7579c9-kjdng 1300m (2%) 1800m (3%) 550Mi (0%) 1200Mi (1%) 73d
rdf-streaming-updater flink-session-cluster-main-taskmanager-6cdd5c87b7-kd5fp 1200m (2%) 4500m (9%) 2152857600 (2%) 2624288k (2%) 32d
recommendation-api recommendation-api-production-64d668997b-nmlsh 700m (1%) 2100m (4%) 350Mi (0%) 900Mi (0%) 73d
shellbox-constraints shellbox-main-64999bc485-djmbd 700m (1%) 2700m (5%) 1524Mi (1%) 3148Mi (3%) 25d
shellbox-media shellbox-main-5c56fb99cc-nblgf 3600m (7%) 4700m (9%) 1524Mi (1%) 3148Mi (3%) 25d
shellbox-syntaxhighlight shellbox-main-7795bf4db7-b5cbs 2100m (4%) 3700m (7%) 1524Mi (1%) 3148Mi (3%) 25d
shellbox-timeline shellbox-main-6796665947-n5cr9 2100m (4%) 3700m (7%) 1524Mi (1%) 3148Mi (3%) 25d
shellbox shellbox-main-7765d7c9db-l4wt5 700m (1%) 2700m (5%) 1524Mi (1%) 3148Mi (3%) 25d
termbox termbox-production-6f5b9d8cf-tlmjf 400m (0%) 1600m (3%) 340Mi (0%) 900Mi (0%) 6d15
h
wikifeeds wikifeeds-production-84d464cb98-74dzk 1600m (3%) 3600m (7%) 1200Mi (1%) 1900Mi (1%) 69d
zotero zotero-production-685c4f4cb6-cmckq 400m (0%) 1500m (3%) 300Mi (0%) 2548Mi (2%) 73d
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 32 (66%) 78400m (163%)
memory 23073920Ki (23%) 54859467008 (55%)
ephemeral-storage 0 (0%) 0 (0%)
Events: <none>
root@deploy1002:~# ping kubernetes1014
PING kubernetes1014(kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230)) 56 data bytes
64 bytes from kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230): icmp_seq=1 ttl=63 time=0.230 ms
64 bytes from kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230): icmp_seq=2 ttl=63 time=0.252 ms
64 bytes from kubernetes1014.eqiad.wmnet (2620:0:861:107:10:64:48:230): icmp_seq=3 ttl=63 time=0.253 ms
^C
--- kubernetes1014 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 10ms
rtt min/avg/max/mdev = 0.230/0.245/0.253/0.010 ms
root@deploy1002:~# kubectl drain kubernetes1014.eqiad.wmnet
node/kubernetes1014.eqiad.wmnet cordoned
error: unable to drain node "kubernetes1014.eqiad.wmnet", aborting command...
There are pending nodes to be drained:
kubernetes1014.eqiad.wmnet
cannot delete Pods with local storage (use --delete-local-data to override): api-gateway/api-gateway-production-5889589fc-dz2p8, shellbox-constraints/shel
lbox-main-64999bc485-djmbd, shellbox-media/shellbox-main-5c56fb99cc-nblgf, shellbox-syntaxhighlight/shellbox-main-7795bf4db7-b5cbs, shellbox-timeline/shel
lbox-main-6796665947-n5cr9, shellbox/shellbox-main-7765d7c9db-l4wt5
cannot delete DaemonSet-managed Pods (use --ignore-daemonsets to ignore): kube-system/calico-node-86v2g
root@deploy1002:~# kubectl drain kubernetes1014.eqiad.wmnet --delete-local-data --ignore-daemonsets
node/kubernetes1014.eqiad.wmnet already cordoned
WARNING: ignoring DaemonSet-managed Pods: kube-system/calico-node-86v2g
evicting pod "blubberoid-production-66947dddf9-4df7f"
evicting pod "eventgate-analytics-production-7b8ccf799b-gwhzj"
evicting pod "zotero-production-685c4f4cb6-cmckq"
evicting pod "apertium-production-6b99ff7694-8d6pw"
evicting pod "changeprop-production-b5fbcffbb-x9q7n"
evicting pod "eventgate-analytics-production-7b8ccf799b-twbq8"
evicting pod "api-gateway-production-5889589fc-dz2p8"
evicting pod "changeprop-production-78fb8b7cd4-hfpwf"
evicting pod "shellbox-main-64999bc485-djmbd"
evicting pod "eventgate-logging-external-production-99fd8cc75-m2l47"
evicting pod "mobileapps-production-746f7579c9-8ncx6"
evicting pod "shellbox-main-7765d7c9db-l4wt5"
evicting pod "eventgate-main-production-fb5d896b9-rdxsn"
evicting pod "mathoid-production-596cb7c944-wmhrs"
evicting pod "termbox-production-6f5b9d8cf-tlmjf"
evicting pod "coredns-584d8549b5-hr6tg"
evicting pod "eventgate-analytics-external-production-7577b69644-lzzhx"
evicting pod "mobileapps-production-746f7579c9-ck7bl"
evicting pod "shellbox-main-5c56fb99cc-nblgf"
evicting pod "eventstreams-internal-main-5bffd6d7c6-t2xkt"
evicting pod "cxserver-production-694c7db9c-6zkjh"
evicting pod "shellbox-main-7795bf4db7-b5cbs"
evicting pod "mobileapps-production-746f7579c9-kjdng"
evicting pod "mobileapps-production-746f7579c9-7wx27"
evicting pod "mobileapps-production-746f7579c9-79hdk"
evicting pod "eventstreams-production-7b64d578d8-nzhzq"
evicting pod "shellbox-main-6796665947-n5cr9"
evicting pod "wikifeeds-production-84d464cb98-74dzk"
evicting pod "mobileapps-production-746f7579c9-58b7q"
evicting pod "mobileapps-production-746f7579c9-96xk7"
evicting pod "recommendation-api-production-64d668997b-nmlsh"
evicting pod "flink-session-cluster-main-taskmanager-6cdd5c87b7-kd5fp"
evicting pod "linkrecommendation-internal-58fd7dd4d8-vvdh5"
evicting pod "mathoid-production-596cb7c944-v6brp"
pod/api-gateway-production-5889589fc-dz2p8 evicted
pod/mobileapps-production-746f7579c9-7wx27 evicted
pod/changeprop-production-b5fbcffbb-x9q7n evicted
pod/cxserver-production-694c7db9c-6zkjh evicted
pod/eventgate-analytics-production-7b8ccf799b-gwhzj evicted
pod/flink-session-cluster-main-taskmanager-6cdd5c87b7-kd5fp evicted
pod/eventgate-analytics-production-7b8ccf799b-twbq8 evicted
pod/mobileapps-production-746f7579c9-kjdng evicted
pod/mobileapps-production-746f7579c9-79hdk evicted
pod/mobileapps-production-746f7579c9-58b7q evicted
pod/eventgate-main-production-fb5d896b9-rdxsn evicted
pod/mathoid-production-596cb7c944-v6brp evicted
pod/eventstreams-production-7b64d578d8-nzhzq evicted
pod/mobileapps-production-746f7579c9-8ncx6 evicted
pod/apertium-production-6b99ff7694-8d6pw evicted
pod/shellbox-main-7795bf4db7-b5cbs evicted
pod/eventgate-analytics-external-production-7577b69644-lzzhx evicted
pod/zotero-production-685c4f4cb6-cmckq evicted
pod/eventgate-logging-external-production-99fd8cc75-m2l47 evicted
pod/blubberoid-production-66947dddf9-4df7f evicted
pod/mathoid-production-596cb7c944-wmhrs evicted
pod/shellbox-main-7765d7c9db-l4wt5 evicted
pod/shellbox-main-6796665947-n5cr9 evicted
pod/coredns-584d8549b5-hr6tg evicted
pod/shellbox-main-64999bc485-djmbd evicted
pod/mobileapps-production-746f7579c9-96xk7 evicted
pod/changeprop-production-78fb8b7cd4-hfpwf evicted
pod/wikifeeds-production-84d464cb98-74dzk evicted
pod/shellbox-main-5c56fb99cc-nblgf evicted
pod/mobileapps-production-746f7579c9-ck7bl evicted
pod/linkrecommendation-internal-58fd7dd4d8-vvdh5 evicted
pod/eventstreams-internal-main-5bffd6d7c6-t2xkt evicted
pod/termbox-production-6f5b9d8cf-tlmjf evicted
pod/recommendation-api-production-64d668997b-nmlsh evicted
node/kubernetes1014.eqiad.wmnet evicted
root@deploy1002:~#

File Metadata

Mime Type
text/plain
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
9342490
Default Alt Text
kubernetes1014.log (14 KB)

Event Timeline

Content licensed under Creative Commons Attribution-ShareAlike (CC BY-SA) 4.0 unless otherwise noted; code licensed under GNU General Public License (GPL) 2.0 or later and other open source licenses. By using this site, you agree to the Terms of Use, Privacy Policy, and Code of Conduct. · Wikimedia Foundation · Privacy Policy · Code of Conduct · Terms of Use · Disclaimer · CC-BY-SA · GPL · Credits