[root@ip-10-151-21-17 securiti-appliance-installer]# journalctl -u k3s-agent --no-pager --since "2026-03-13 10:35:19" Mar 13 10:35:51 ip-10-151-21-17.us-west-2.compute.internal systemd[1]: Starting Lightweight Kubernetes... Mar 13 10:35:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:51Z" level=info msg="Acquiring lock file /mnt/rancher/k3s/data/.lock" Mar 13 10:35:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:51Z" level=info msg="Preparing data dir /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash nsenter is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash pigz is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash runc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash conntrack is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/ebtables-legacy is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/ebtablesu is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/iptables-apply is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/iptables-detect.sh is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/nft is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/xtables-nft-multi is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash blkid is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash busybox is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash cni is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash ip is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash ipset is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash k3s is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash losetup is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash slirp4netns is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/ebtablesd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash coreutils is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash find is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash fuse-overlayfs is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash xargs is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/xtables-legacy-multi is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash aux/xtables-set-mode.sh is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash check-config is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash containerd-shim-runc-v2 is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified hash ethtool is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link setlogcons is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unpigz is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-nft-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link firewall is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nameif is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link run-parts is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link base32 is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dircolors is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fgrep is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link gzip is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ifdown is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link paste is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link basename is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chcon is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link crontab is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fold is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link gunzip is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link hexedit is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link insmod is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ipneigh is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ebtables is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link free is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fuser is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lsattr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nl is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link linuxrc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lspci is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nproc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link login is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mknod is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link readlink is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link setkeycodes is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sha1sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ts is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link deluser is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fallocate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ifconfig is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mdev is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mkfifo is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link run-init is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link shuf is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link udhcpc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link [[ is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link linux32 is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link passwd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-legacy-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link bzcat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link cmp is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link deallocvt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lsof is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link od is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ping is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link stat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link setarch is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unix2dos is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link vi is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-legacy-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-translate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nuke is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link umount is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ash is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link hostid is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mkdir is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link vlock is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chroot is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fsfreeze is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link linux64 is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lzma is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-restore-translate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dir is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link i2ctransfer is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mkpasswd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ps is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sha384sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nslookup is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sulogin is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link adduser is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link resume is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link uname is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ebtables-nft-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link awk is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link cksum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link k3s-completion is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link pivot_root is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ptx is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link rm is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link uevent is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link k3s-etcd-snapshot is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link md5sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link runlevel is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link setconsole is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link touch is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link watchdog is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link which is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link id is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ifup is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ipaddr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link iplink is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link pipe_progress is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sha256sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link wc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link csplit is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link devmem is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link i2cdump is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link k3s-token is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lsusb is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link su is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link install is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mv is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link setpriv is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link clear is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mim is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link pidof is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-apply is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-nft-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-nft-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link cpio is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link klogd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link containerd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link netstat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link watch is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link [ is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/arptables is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link base64 is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link kubectl is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link arp is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ls is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link truncate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tsort is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/modprobe is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link bandwidth is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link cp is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link microcom is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link numfmt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link reset is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link strings is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tftp is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link bunzip2 is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link flannel is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link pathchk is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link uuencode is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link xxd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/arptables-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link expr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link iptunnel is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link loadfont is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sha224sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link shred is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link svok is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link uptime is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-nft is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link last is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lzopcat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sleep is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link yes is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link crictl is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link cut is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link factor is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sync is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link comm is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link eject is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nice is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unexpand is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dirname is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-nft-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-translate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fmt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/arptables-nft-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link head is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lzcat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link runcon is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sysctl is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tar is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link true is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link who is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dnsdomainname is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link host-local is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link xz is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link xzcat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link zcat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ebtables-nft is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-legacy is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link du is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link reboot is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ebtables-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link b2sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lsscsi is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link partprobe is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link uniq is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link addgroup is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link freeramdisk is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link hdparm is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ctr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-legacy is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link getfattr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link getopt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link halt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link pwd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link top is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link env is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fdflush is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link join is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link killall5 is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sort is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ubirename is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-legacy-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link cat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link false is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link killall is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unlzop is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mkdosfs is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mktemp is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link setserial is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link basenc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chrt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chvt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link k3s-secrets-encrypt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mesg is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unlink is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chattr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link egrep is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link logname is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link wget is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dumpkmap is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link echo is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link i2cdetect is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link printenv is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link setfattr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link time is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-restore-translate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ebtables-translate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/mount is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link getty is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link k3s-certificate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link loopback is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mke2fs is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link seq is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tail is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/xtables-monitor is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chmod is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link df is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link realpath is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link start-stop-daemon is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/arptables-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/arptables-nft-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link delgroup is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fsck is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link lsmod is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nohup is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/arptables-nft is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fbset is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ipcs is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link split is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ether-wake is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link bc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link diff is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dnsd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dos2unix is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link pinky is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link pr is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link route is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ebtables-nft-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link hwclock is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link kill is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link logger is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link nologin is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link patch is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sh is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sha3sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link arping is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link bridge is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chgrp is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link dd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link fdformat is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ipcrm is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link less is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link makedevs is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables-legacy-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link chown is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link printf is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link resize is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link switch_root is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tac is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tee is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unzip is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link grep is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link groups is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link hostname is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link inetd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link k3s-agent is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link portmap is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link users is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link uudecode is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ebtables-restore is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link date is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link link is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link stty is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link vdir is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-save is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link expand is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link poweroff is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link svc is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/arptables-translate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link init is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link iproute is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link syslogd is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unxz is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link vconfig is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link i2cget is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link k3s-server is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link more is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sha512sum is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link test is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ar is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/ip6tables is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link i2cset is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link iprule is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link rdate is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link traceroute is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link aux/iptables-nft is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link crond is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link ln is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link rmmod is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link telnet is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link timeout is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link unlzma is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link loadkmap is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link mountpoint is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link openvt is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link usleep is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link whoami is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link arch is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link rmdir is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link sed is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link tty is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Verified link w is correct" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/cni -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/bandwidth -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/bridge -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/firewall -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/flannel -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/host-local -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/loopback -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Creating symlink /mnt/rancher/k3s/data/cni/portmap -> /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/cni" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Asset dir /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Running /mnt/rancher/k3s/data/4a9584f4b72a328dc77a958636e91442257c1890aaa66f0508b3604bf676dbdd/bin/k3s-agent [/usr/bin/k3s agent]" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=info msg="Starting k3s agent v1.34.4+k3s1 (c6017918)" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=info msg="Updated load balancer k3s-agent-load-balancer default server: 10.151.22.85:6443" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=info msg="Running load balancer k3s-agent-load-balancer 127.0.0.1:6444 -> [] [default: 10.151.22.85:6443]" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=debug msg="Supervisor proxy using supervisor=https://127.0.0.1:6444 apiserver=https://127.0.0.1:6444 lb=true" Mar 13 10:35:53 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:53Z" level=warning msg="Cluster CA certificate is not trusted by the host CA bundle, but the token does not include a CA hash. Use the full token from the server's node-token file to enable Cluster CA validation." Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Using private registry config file at /etc/rancher/k3s/registries.yaml" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Module overlay was already loaded" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Module nf_conntrack was already loaded" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Module br_netfilter was already loaded" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Module iptable_nat was already loaded" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Module iptable_filter was already loaded" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=debug msg="getConntrackMax: using scaled conntrack-max-per-core" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=error msg="Distributed registry mirror skipping invalid registry: localhost" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Starting distributed registry mirror at https://10.151.21.17:6443/v2 for registries [251337915507.dkr.ecr.us-west-2.amazonaws.com docker.io]" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="Starting distributed registry P2P node at 10.151.21.17:5000" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.490Z INFO dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:322 starting refreshing cpl 0 with key CIQAAAHRUMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA (routing table size was 0) Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.490Z WARN dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:187 failed when refreshing routing tablefailed to query for self, err=failed to find any peer in table; failed to refresh cpl=0, err=failed to find any peer in table Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="starting p2p router" id=QmXbGzvs4rzxkGwMVGJFZemoksNqf2EP4BnYBcMX9Lwokq logger=spegel/p2p Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=debug msg="failed to run bootstrap" attempts=1 error="client not ready" logger=spegel/p2p Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:56.492124 6408 secure_serving.go:211] Serving securely on [::]:6443 Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:56.492125 6408 dynamic_cafile_content.go:161] "Starting controller" name="client-ca-bundle::/mnt/rancher/k3s/agent/client-ca.crt" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:56.492204 6408 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/mnt/rancher/k3s/agent/serving-kubelet.crt::/mnt/rancher/k3s/agent/serving-kubelet.key" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:56.492204 6408 tlsconfig.go:243] "Starting DynamicServingCertificateController" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=warning msg="Cluster CA certificate is not trusted by the host CA bundle, but the token does not include a CA hash. Use the full token from the server's node-token file to enable Cluster CA validation." Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=debug msg="Failed to update P2P address annotations and labels: nodes \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=debug msg="failed to run bootstrap" attempts=2 error="routing table is empty after bootstrapping" logger=spegel/p2p Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.546Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiCJeFjR9DNQCVq1r2ih1TFYQB7ALwfZiPwh1Vj2ETGohg=="} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.546Z ERROR dht zap@v1.27.1/sugar.go:257 Ignored key without a value. {"ignored": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.546Z INFO dht go-libp2p-kad-dht@v0.36.0/dht.go:761 no closer peers to send Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.546Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiCJeFjR9DNQCVq1r2ih1TFYQB7ALwfZiPwh1Vj2ETGohg==", "time": 0.000081415} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.546Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiCJeFjR9DNQCVq1r2ih1TFYQB7ALwfZiPwh1Vj2ETGohg==", "time": 0.000114679} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.546Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw=="} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z ERROR dht zap@v1.27.1/sugar.go:257 Ignored key without a value. {"ignored": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z INFO dht go-libp2p-kad-dht@v0.36.0/dht.go:761 no closer peers to send Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000065819} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000077122} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAABQwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAABQwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000035781} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAABQwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000046434} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.547Z INFO dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:322 starting refreshing cpl 0 with key CIQAAAMII4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA (routing table size was 1) Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.548Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.548Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.548Z INFO dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:331 finished refreshing cpl 0, routing table size is now 1 Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.548Z INFO dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:322 starting refreshing cpl 1 with key CIQAAACXKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA (routing table size was 1) Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.549Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.549Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.549Z INFO dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:331 finished refreshing cpl 1, routing table size is now 1 Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.549Z INFO dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:322 starting refreshing cpl 2 with key CIQAAAEA24AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA (routing table size was 1) Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.550Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.550Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.550Z INFO dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:331 finished refreshing cpl 2, routing table size is now 1 Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.551Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAeAVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.551Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAeAVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000032821} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.551Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAeAVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000044241} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.551Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAK7iAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.551Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAK7iAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000057409} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.551Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAK7iAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000071812} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.556Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.556Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.556Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 0 as time since last refresh not above interval Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.556Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 1 as time since last refresh not above interval Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.556Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 2 as time since last refresh not above interval Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=debug msg="failed to run bootstrap" attempts=3 error="bootstrap completed but connectivity has not been reached" logger=spegel/p2p Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.591Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.591Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.592Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.592Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.593Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.593Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.593Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.593Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.594Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:35:56.594Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:35:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:56Z" level=info msg="bootstrap completed connectivity is reached" duration=363.304176ms logger=spegel/p2p Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for crun container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Found crun container runtime at /usr/bin/crun" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for nvidia container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for nvidia in $PATH: exec: \"nvidia-container-runtime\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for nvidia-experimental container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for nvidia-experimental in $PATH: exec: \"nvidia-container-runtime-experimental\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for nvidia-cdi container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for nvidia-cdi in $PATH: exec: \"nvidia-container-runtime.cdi\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for wasmedge container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for wasmedge in $PATH: exec: \"containerd-shim-wasmedge-v1\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for wasmer container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for wasmer in $PATH: exec: \"containerd-shim-wasmer-v1\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for wasmtime container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for wasmtime in $PATH: exec: \"containerd-shim-wasmtime-v1\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for lunatic container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for lunatic in $PATH: exec: \"containerd-shim-lunatic-v1\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for slight container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for slight in $PATH: exec: \"containerd-shim-slight-v1\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for spin container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for spin in $PATH: exec: \"containerd-shim-spin-v2\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Searching for wws container runtime" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Starting k3s-cert-monitor with monitoring period 30m0s" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Getting list of apiserver endpoints from server" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Error searching for wws in $PATH: exec: \"containerd-shim-wws-v1\": executable file not found in $PATH" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Creating k3s-cert-monitor event broadcaster" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Running k3s-cert-monitor certificate expiration check" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:57.494223 6408 event.go:389] "Event occurred" object="ip-10-151-21-17.us-west-2.compute.internal" fieldPath="" kind="Node" apiVersion="" type="Normal" reason="CertificateExpirationOK" message="Node and Certificate Authority certificates managed by k3s are OK" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Logging containerd to /mnt/rancher/k3s/agent/containerd/containerd.log" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Running containerd -c /mnt/rancher/k3s/agent/etc/containerd/config.toml --log-level debug" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Failed to update P2P address annotations and labels: nodes \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Tunnel authorizer failed to get Kubelet Port: nodes \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.555724748Z" level=info msg="starting containerd" revision= version=v2.1.5-k3s1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560681943Z" level=info msg="loading plugin" id=io.containerd.content.v1.content type=io.containerd.content.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560718299Z" level=info msg="loading plugin" id=io.containerd.image-verifier.v1.bindir type=io.containerd.image-verifier.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560725396Z" level=info msg="loading plugin" id=io.containerd.internal.v1.opt type=io.containerd.internal.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560806438Z" level=info msg="loading plugin" id=io.containerd.warning.v1.deprecations type=io.containerd.warning.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560812518Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.blockfile type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560835181Z" level=info msg="skip loading plugin" error="no scratch file generator: skip plugin" id=io.containerd.snapshotter.v1.blockfile type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560840481Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.btrfs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560980999Z" level=info msg="skip loading plugin" error="path /mnt/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs (xfs) must be a btrfs filesystem to be used with the btrfs snapshotter: skip plugin" id=io.containerd.snapshotter.v1.btrfs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560987944Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.devmapper type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560992747Z" level=info msg="skip loading plugin" error="devmapper not configured: skip plugin" id=io.containerd.snapshotter.v1.devmapper type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.560996562Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.erofs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.561094469Z" level=info msg="skip loading plugin" error="EROFS unsupported, please `modprobe erofs`: skip plugin" id=io.containerd.snapshotter.v1.erofs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.561099963Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.native type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.561133037Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.overlayfs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.562878458Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.fuse-overlayfs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.562925108Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.zfs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.562941368Z" level=info msg="skip loading plugin" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.zfs: no such file or directory: skip plugin" id=io.containerd.snapshotter.v1.zfs type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.562950952Z" level=info msg="loading plugin" id=io.containerd.event.v1.exchange type=io.containerd.event.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.562977014Z" level=info msg="loading plugin" id=io.containerd.monitor.task.v1.cgroups type=io.containerd.monitor.task.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.563152752Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.stargz type=io.containerd.snapshotter.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.566833226Z" level=info msg="loading plugin" id=io.containerd.metadata.v1.bolt type=io.containerd.metadata.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.566873709Z" level=info msg="metadata content store policy set" policy=shared Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Got apiserver addresses from supervisor: [10.151.22.85:6443]" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Server 10.151.22.85:6443@STANDBY*->UNCHECKED from add to load balancer k3s-agent-load-balancer" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Updated load balancer k3s-agent-load-balancer server addresses -> [10.151.22.85:6443] [default: 10.151.22.85:6443]" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Set health check for load balancer k3s-agent-load-balancer: 10.151.22.85:6443" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Connecting to proxy" url="wss://10.151.22.85:6443/v1-k3s/connect" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Server 10.151.22.85:6443@UNCHECKED*->RECOVERING from successful dial" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Syncing apiserver addresses from tunnel watch: [10.151.22.85:6443]" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Connected to proxy" url="wss://10.151.22.85:6443/v1-k3s/connect" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Remotedialer connected to proxy" url="wss://10.151.22.85:6443/v1-k3s/connect" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572842572Z" level=info msg="loading plugin" id=io.containerd.gc.v1.scheduler type=io.containerd.gc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572874990Z" level=info msg="loading plugin" id=io.containerd.differ.v1.erofs type=io.containerd.differ.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572911278Z" level=info msg="skip loading plugin" error="could not find mkfs.erofs: exec: \"mkfs.erofs\": executable file not found in $PATH: skip plugin" id=io.containerd.differ.v1.erofs type=io.containerd.differ.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572917685Z" level=info msg="loading plugin" id=io.containerd.differ.v1.walking type=io.containerd.differ.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572929914Z" level=info msg="loading plugin" id=io.containerd.lease.v1.manager type=io.containerd.lease.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572939414Z" level=info msg="loading plugin" id=io.containerd.service.v1.containers-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572946807Z" level=info msg="loading plugin" id=io.containerd.service.v1.content-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572971757Z" level=info msg="loading plugin" id=io.containerd.service.v1.diff-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572989405Z" level=info msg="loading plugin" id=io.containerd.service.v1.images-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.572998864Z" level=info msg="loading plugin" id=io.containerd.service.v1.introspection-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573008791Z" level=info msg="loading plugin" id=io.containerd.service.v1.namespaces-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573017618Z" level=info msg="loading plugin" id=io.containerd.service.v1.snapshots-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573022859Z" level=info msg="loading plugin" id=io.containerd.shim.v1.manager type=io.containerd.shim.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573038368Z" level=info msg="loading plugin" id=io.containerd.runtime.v2.task type=io.containerd.runtime.v2 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573107257Z" level=info msg="loading plugin" id=io.containerd.service.v1.tasks-service type=io.containerd.service.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573116150Z" level=debug msg="No blockio config file specified, blockio not configured" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573119533Z" level=debug msg="No RDT config file specified, RDT not configured" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573124626Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.containers type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573132813Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.content type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573144568Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.diff type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573163923Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.events type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573169625Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.images type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573177885Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.introspection type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573207381Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.leases type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573214960Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.namespaces type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573221309Z" level=info msg="loading plugin" id=io.containerd.sandbox.store.v1.local type=io.containerd.sandbox.store.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573227053Z" level=info msg="loading plugin" id=io.containerd.transfer.v1.local type=io.containerd.transfer.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573245589Z" level=info msg="loading plugin" id=io.containerd.cri.v1.images type=io.containerd.cri.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573319098Z" level=info msg="Get image filesystem path \"/mnt/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs\" for snapshotter \"overlayfs\"" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573331275Z" level=info msg="Start snapshots syncer" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573351293Z" level=info msg="loading plugin" id=io.containerd.cri.v1.runtime type=io.containerd.cri.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573589898Z" level=info msg="starting cri plugin" config="{\"containerd\":{\"defaultRuntimeName\":\"runc\",\"runtimes\":{\"crun\":{\"runtimeType\":\"io.containerd.runc.v2\",\"runtimePath\":\"\",\"PodAnnotations\":null,\"ContainerAnnotations\":null,\"options\":{\"BinaryName\":\"/usr/bin/crun\",\"SystemdCgroup\":true},\"privileged_without_host_devices\":false,\"privileged_without_host_devices_all_devices_allowed\":false,\"cgroupWritable\":false,\"baseRuntimeSpec\":\"\",\"cniConfDir\":\"\",\"cniMaxConfNum\":0,\"snapshotter\":\"\",\"sandboxer\":\"podsandbox\",\"io_type\":\"\"},\"runc\":{\"runtimeType\":\"io.containerd.runc.v2\",\"runtimePath\":\"\",\"PodAnnotations\":null,\"ContainerAnnotations\":null,\"options\":{\"BinaryName\":\"\",\"CriuImagePath\":\"\",\"CriuWorkPath\":\"\",\"IoGid\":0,\"IoUid\":0,\"NoNewKeyring\":false,\"Root\":\"\",\"ShimCgroup\":\"\",\"SystemdCgroup\":true},\"privileged_without_host_devices\":false,\"privileged_without_host_devices_all_devices_allowed\":false,\"cgroupWritable\":false,\"baseRuntimeSpec\":\"\",\"cniConfDir\":\"\",\"cniMaxConfNum\":0,\"snapshotter\":\"\",\"sandboxer\":\"podsandbox\",\"io_type\":\"\"},\"runhcs-wcow-process\":{\"runtimeType\":\"io.containerd.runhcs.v1\",\"runtimePath\":\"\",\"PodAnnotations\":null,\"ContainerAnnotations\":null,\"options\":null,\"privileged_without_host_devices\":false,\"privileged_without_host_devices_all_devices_allowed\":false,\"cgroupWritable\":false,\"baseRuntimeSpec\":\"\",\"cniConfDir\":\"\",\"cniMaxConfNum\":0,\"snapshotter\":\"\",\"sandboxer\":\"podsandbox\",\"io_type\":\"\"}},\"ignoreBlockIONotEnabledErrors\":false,\"ignoreRdtNotEnabledErrors\":false},\"cni\":{\"binDir\":\"\",\"binDirs\":[\"/mnt/rancher/k3s/data/cni\"],\"confDir\":\"/mnt/rancher/k3s/agent/etc/cni/net.d\",\"maxConfNum\":1,\"setupSerially\":false,\"confTemplate\":\"\",\"ipPref\":\"\",\"useInternalLoopback\":false},\"enableSelinux\":false,\"selinuxCategoryRange\":1024,\"maxContainerLogLineSize\":16384,\"disableApparmor\":false,\"restrictOOMScoreAdj\":false,\"disableProcMount\":false,\"unsetSeccompProfile\":\"\",\"tolerateMissingHugetlbController\":true,\"disableHugetlbController\":true,\"device_ownership_from_security_context\":false,\"ignoreImageDefinedVolumes\":false,\"netnsMountsUnderStateDir\":false,\"enableUnprivilegedPorts\":true,\"enableUnprivilegedICMP\":true,\"enableCDI\":true,\"cdiSpecDirs\":[\"/etc/cdi\",\"/var/run/cdi\"],\"drainExecSyncIOTimeout\":\"0s\",\"ignoreDeprecationWarnings\":null,\"containerdRootDir\":\"/mnt/rancher/k3s/agent/containerd\",\"containerdEndpoint\":\"/run/k3s/containerd/containerd.sock\",\"rootDir\":\"/mnt/rancher/k3s/agent/containerd/io.containerd.grpc.v1.cri\",\"stateDir\":\"/run/k3s/containerd/io.containerd.grpc.v1.cri\"}" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573630095Z" level=info msg="loading plugin" id=io.containerd.podsandbox.controller.v1.podsandbox type=io.containerd.podsandbox.controller.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573672443Z" level=info msg="loading plugin" id=io.containerd.sandbox.controller.v1.shim type=io.containerd.sandbox.controller.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573722971Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.sandbox-controllers type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573737132Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.sandboxes type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573743694Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.snapshots type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573749409Z" level=info msg="loading plugin" id=io.containerd.streaming.v1.manager type=io.containerd.streaming.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573758488Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.streaming type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573763748Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.tasks type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573769502Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.transfer type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573774785Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.version type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573783563Z" level=info msg="loading plugin" id=io.containerd.monitor.container.v1.restart type=io.containerd.monitor.container.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573805951Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.healthcheck type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573811305Z" level=info msg="loading plugin" id=io.containerd.nri.v1.nri type=io.containerd.nri.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573822152Z" level=info msg="runtime interface created" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573824733Z" level=info msg="created NRI interface" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573829008Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.cri type=io.containerd.grpc.v1 Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573842804Z" level=info msg="Connect containerd service" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.573854064Z" level=info msg="using experimental NRI integration - disable nri plugin to prevent this" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.574202307Z" level=error msg="failed to load cni during init, please check CRI plugin status before setting up network for pods" error="cni config load failed: no network config found in /mnt/rancher/k3s/agent/etc/cni/net.d: cni plugin not initialized: failed to load cni config" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.581171236Z" level=debug msg="runtime \"runc\" supports recursive read-only mounts" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.581182436Z" level=debug msg="runtime \"runc\" supports CRI userns: true" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.586984006Z" level=debug msg="runtime \"crun\" supports recursive read-only mounts" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.586995974Z" level=debug msg="runtime \"crun\" supports CRI userns: true" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587003936Z" level=debug msg="failed to introspect features of runtime \"runhcs-wcow-process\"" error="introspecting OCI runtime features needs the runtime type to be \"io.containerd.runc.v2\", got \"io.containerd.runhcs.v1\"" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587104377Z" level=info msg="Start subscribing containerd event" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587130864Z" level=info msg="Start recovering state" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587205253Z" level=info msg="Start event monitor" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587209190Z" level=info msg=serving... address=/run/k3s/containerd/containerd.sock.ttrpc Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587218521Z" level=info msg="Start cni network conf syncer for default" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587229976Z" level=info msg="Start streaming server" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587233425Z" level=info msg=serving... address=/run/k3s/containerd/containerd.sock Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587239534Z" level=info msg="Registered namespace \"k8s.io\" with NRI" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587244176Z" level=info msg="runtime interface starting up..." Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587247604Z" level=info msg="starting plugins..." Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587253675Z" level=info msg="Synchronizing NRI (plugin) with current runtime state" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587315405Z" level=debug msg="sd notification" notified=false state="READY=1" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57.587325469Z" level=info msg="containerd successfully booted in 0.032247s" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=info msg="Running kube-proxy --cluster-cidr=10.42.0.0/16 --conntrack-max-per-core=0 --conntrack-tcp-timeout-close-wait=0s --conntrack-tcp-timeout-established=0s --healthz-bind-address=127.0.0.1 --hostname-override=ip-10-151-21-17.us-west-2.compute.internal --kubeconfig=/mnt/rancher/k3s/agent/kubeproxy.kubeconfig --proxy-mode=iptables" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Syncing apiserver addresses from server: [10.151.22.85:6443]" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:57Z" level=debug msg="Cancelled apiserver addresses sync: [10.151.22.85:6443]" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:57.722572 6408 shared_informer.go:349] "Waiting for caches to sync" controller="node informer cache" Mar 13 10:35:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:57.822969 6408 shared_informer.go:356] "Caches are synced" controller="node informer cache" Mar 13 10:35:58 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=info msg="Server 10.151.22.85:6443@RECOVERING*->ACTIVE from successful health check" Mar 13 10:35:58 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=info msg="containerd is now running" Mar 13 10:35:58 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=debug msg="Image import watcher created" Mar 13 10:35:58 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=debug msg="Failed to update P2P address annotations and labels: nodes \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:58 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=debug msg="Tunnel authorizer failed to get Kubelet Port: nodes \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:58 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=debug msg="Starting embedded registry image state tracker" Mar 13 10:35:58 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=debug msg="Kubelet image credential provider bin directory check failed: stat /var/lib/rancher/credentialprovider/bin: no such file or directory" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:58Z" level=info msg="Running kubelet --config-dir=/mnt/rancher/k3s/agent/etc/kubelet.conf.d --containerd=/run/k3s/containerd/containerd.sock --hostname-override=ip-10-151-21-17.us-west-2.compute.internal --kubeconfig=/mnt/rancher/k3s/agent/kubelet.kubeconfig --node-ip=10.151.21.17 --node-labels=role=worker,node-role.appliance.securiti.io/worker=true --read-only-port=0" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: Flag --containerd has been deprecated, This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed. Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="waiting for store events" logger=spegel Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.002981 6408 server.go:525] "Kubelet version" kubeletVersion="v1.34.4+k3s1" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.002997 6408 server.go:527] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK="" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.003014 6408 watchdog_linux.go:95] "Systemd watchdog is not enabled" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.003019 6408 watchdog_linux.go:137] "Systemd watchdog is not enabled or the interval is invalid, so health checking will not be started." Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.004089 6408 dynamic_cafile_content.go:161] "Starting controller" name="client-ca-bundle::/mnt/rancher/k3s/agent/client-ca.crt" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.005962 6408 server.go:1419] "Using cgroup driver setting received from the CRI runtime" cgroupDriver="systemd" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011219 6408 server.go:777] "--cgroups-per-qos enabled, but --cgroup-root was not specified. Defaulting to /" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011245 6408 server.go:838] "NoSwap is set due to memorySwapBehavior not specified" memorySwapBehavior="" FailSwapOn=false Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011348 6408 container_manager_linux.go:270] "Container manager verified user specified cgroup-root exists" cgroupRoot=[] Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011360 6408 container_manager_linux.go:275] "Creating Container Manager object based on Node Config" nodeConfig={"NodeName":"ip-10-151-21-17.us-west-2.compute.internal","RuntimeCgroupsName":"","SystemCgroupsName":"","KubeletCgroupsName":"","KubeletOOMScoreAdj":-999,"ContainerRuntime":"","CgroupsPerQOS":true,"CgroupRoot":"/","CgroupDriver":"systemd","KubeletRootDir":"/var/lib/kubelet","ProtectKernelDefaults":false,"KubeReservedCgroupName":"","SystemReservedCgroupName":"","ReservedSystemCPUs":{},"EnforceNodeAllocatable":{"pods":{}},"KubeReserved":null,"SystemReserved":null,"HardEvictionThresholds":[{"Signal":"imagefs.available","Operator":"LessThan","Value":{"Quantity":null,"Percentage":0.05},"GracePeriod":0,"MinReclaim":null},{"Signal":"memory.available","Operator":"LessThan","Value":{"Quantity":"200Mi","Percentage":0},"GracePeriod":0,"MinReclaim":null},{"Signal":"nodefs.available","Operator":"LessThan","Value":{"Quantity":null,"Percentage":0.05},"GracePeriod":0,"MinReclaim":null}],"QOSReserved":{},"CPUManagerPolicy":"none","CPUManagerPolicyOptions":null,"TopologyManagerScope":"container","CPUManagerReconcilePeriod":10000000000,"MemoryManagerPolicy":"None","MemoryManagerReservedMemory":null,"PodPidsLimit":-1,"EnforceCPULimits":true,"CPUCFSQuotaPeriod":100000000,"TopologyManagerPolicy":"none","TopologyManagerPolicyOptions":null,"CgroupVersion":2} Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011511 6408 topology_manager.go:138] "Creating topology manager with none policy" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011517 6408 container_manager_linux.go:306] "Creating device plugin manager" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011533 6408 container_manager_linux.go:315] "Creating Dynamic Resource Allocation (DRA) manager" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011653 6408 state_mem.go:36] "Initialized new in-memory state store" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011750 6408 kubelet.go:475] "Attempting to sync node with API server" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011770 6408 kubelet.go:376] "Adding static pod path" path="/mnt/rancher/k3s/agent/pod-manifests" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011786 6408 kubelet.go:387] "Adding apiserver pod source" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.011794 6408 apiserver.go:42] "Waiting for node sync before watching apiserver pods" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.012385 6408 kuberuntime_manager.go:291] "Container runtime initialized" containerRuntime="containerd" version="v2.1.5-k3s1" apiVersion="v1" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.012607 6408 kubelet.go:940] "Not starting ClusterTrustBundle informer because we are in static kubelet mode or the ClusterTrustBundleProjection featuregate is disabled" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.012625 6408 kubelet.go:964] "Not starting PodCertificateRequest manager because we are in static kubelet mode or the PodCertificateProjection feature gate is disabled" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.012955 6408 server.go:1257] "Started kubelet" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.012997 6408 ratelimit.go:56] "Setting rate limiting for endpoint" service="podresources" qps=100 burstTokens=10 Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013026 6408 server_v1.go:49] "podresources" method="list" useActivePods=true Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013001 6408 server.go:180] "Starting to listen" address="0.0.0.0" port=10250 Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013113 6408 server.go:249] "Starting to serve the podresources API" endpoint="unix:/var/lib/kubelet/pod-resources/kubelet.sock" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013572 6408 fs_resource_analyzer.go:67] "Starting FS ResourceAnalyzer" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013608 6408 dynamic_serving_content.go:135] "Starting controller" name="kubelet-server-cert-files::/mnt/rancher/k3s/agent/serving-kubelet.crt::/mnt/rancher/k3s/agent/serving-kubelet.key" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.013695 6408 kubelet_node_status.go:404] "Error getting the current node from lister" err="node \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013717 6408 volume_manager.go:313] "Starting Kubelet Volume Manager" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013766 6408 desired_state_of_world_populator.go:146] "Desired state populator starts to run" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.013933 6408 server.go:310] "Adding debug handlers to kubelet server" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.013974 6408 kubelet.go:1615] "Image garbage collection failed once. Stats initialization may not have completed yet" err="invalid capacity 0 on image filesystem" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.014268 6408 factory.go:223] Registration of the systemd container factory successfully Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.014310 6408 reconciler.go:29] "Reconciler: start to sync state" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.014340 6408 factory.go:221] Registration of the crio container factory failed: Get "http://%2Fvar%2Frun%2Fcrio%2Fcrio.sock/info": dial unix /var/run/crio/crio.sock: connect: no such file or directory Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.015494 6408 factory.go:223] Registration of the containerd container factory successfully Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.018487 6408 kubelet_network_linux.go:54] "Initialized iptables rules." protocol="IPv6" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.019104 6408 nodelease.go:49] "Failed to get node when trying to set owner ref to the node lease" err="nodes \"ip-10-151-21-17.us-west-2.compute.internal\" not found" node="ip-10-151-21-17.us-west-2.compute.internal" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025361 6408 cpu_manager.go:221] "Starting CPU manager" policy="none" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025373 6408 cpu_manager.go:222] "Reconciling" reconcilePeriod="10s" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025392 6408 state_mem.go:36] "Initialized new in-memory state store" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025514 6408 state_mem.go:88] "Updated default CPUSet" cpuSet="" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025521 6408 state_mem.go:96] "Updated CPUSet assignments" assignments={} Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025534 6408 policy_none.go:49] "None policy: Start" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025540 6408 memory_manager.go:187] "Starting memorymanager" policy="None" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025547 6408 state_mem.go:36] "Initializing new in-memory state store" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025613 6408 state_mem.go:77] "Updated machine memory state" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025619 6408 policy_none.go:47] "Start" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025869 6408 kubelet_network_linux.go:54] "Initialized iptables rules." protocol="IPv4" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025881 6408 status_manager.go:244] "Starting to sync pod status with apiserver" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.025908 6408 kubelet.go:2428] "Starting kubelet main sync loop" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.025934 6408 kubelet.go:2452] "Skipping pod synchronization" err="[container runtime status check may not have completed yet, PLEG is not healthy: pleg has yet to be successful]" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.027532 6408 manager.go:513] "Failed to read data from checkpoint" err="checkpoint is not found" checkpoint="kubelet_internal_checkpoint" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.027622 6408 eviction_manager.go:189] "Eviction manager: starting control loop" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.027629 6408 container_log_manager.go:146] "Initializing container log rotate workers" workers=1 monitorPeriod="10s" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.027731 6408 plugin_manager.go:118] "Starting Kubelet Plugin Manager" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.028097 6408 eviction_manager.go:267] "eviction manager: failed to check if we have separate container filesystem. Ignoring." err="no imagefs label for configured runtime" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.028126 6408 eviction_manager.go:292] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.128353 6408 kubelet_node_status.go:75] "Attempting to register node" node="ip-10-151-21-17.us-west-2.compute.internal" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.132143 6408 kubelet_node_status.go:78] "Successfully registered node" node="ip-10-151-21-17.us-west-2.compute.internal" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.132167 6408 kubelet_node_status.go:486] "Error updating node status, will retry" err="error getting node \"ip-10-151-21-17.us-west-2.compute.internal\": node \"ip-10-151-21-17.us-west-2.compute.internal\" not found" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="Annotations and labels have been set successfully on node: ip-10-151-21-17.us-west-2.compute.internal" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="Creating the CNI conf in directory /mnt/rancher/k3s/agent/etc/cni/net.d" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="Creating the flannel configuration for backend vxlan in file /mnt/rancher/k3s/agent/etc/flannel/net-conf.json" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="The flannel configuration is {\n\t\"Network\": \"10.42.0.0/16\",\n\t\"EnableIPv6\": false,\n\t\"EnableIPv4\": true,\n\t\"IPv6Network\": \"::/0\",\n\t\"Backend\": {\n\t\"Type\": \"vxlan\"\n}\n}\n" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="Starting flannel with backend vxlan" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59.144662121Z" level=debug msg="ignore event from cni conf dir: CREATE \"/mnt/rancher/k3s/agent/etc/cni/net.d/10-flannel.conflist\"" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59.144690336Z" level=debug msg="receiving change event from cni conf dir: WRITE \"/mnt/rancher/k3s/agent/etc/cni/net.d/10-flannel.conflist\"" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.159958 6408 reflector.go:205] "Failed to watch" err="context canceled" reflector="k8s.io/client-go@v1.34.4-k3s1/tools/cache/reflector.go:290" type="*v1.Node" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="Flannel found PodCIDR assigned for node ip-10-151-21-17.us-west-2.compute.internal" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="k3s agent is up and running" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="No interface defined for flannel in the config. Fetching the default gateway interface" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=debug msg="The interface eth0 will be used by flannel" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal systemd[1]: Started Lightweight Kubernetes. Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="The interface eth0 with ipv4 address 10.151.21.17 will be used by flannel" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.160603 6408 kube.go:139] Waiting 10m0s for node controller to sync Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.160625 6408 kube.go:537] Starting kube subnet manager Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.162966 6408 kube.go:558] Creating the node lease for IPv4. This is the n.Spec.PodCIDRs: [10.42.0.0/24] Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.244027 6408 kuberuntime_manager.go:1828] "Updating runtime config through cri with podcidr" CIDR="10.42.1.0/24" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59.244219255Z" level=debug msg="UpdateRuntimeConfig with config network_config:{pod_cidr:\"10.42.1.0/24\"}" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59.244235439Z" level=info msg="No cni config template is specified, wait for other system components to drop the config." Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59.244239708Z" level=debug msg="UpdateRuntimeConfig returns successfully" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.244508 6408 kubelet_network.go:47] "Updating Pod CIDR" originalPodCIDR="" newPodCIDR="10.42.1.0/24" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.447393 6408 kubelet_node_status.go:439] "Fast updating node status as it just became ready" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="Tunnel authorizer set Kubelet Port 0.0.0.0:10250" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:35:59Z" level=info msg="Node P2P address annotations and labels added: /ip4/10.151.21.17/tcp/5000/p2p/QmXbGzvs4rzxkGwMVGJFZemoksNqf2EP4BnYBcMX9Lwokq {\"ID\":\"QmXbGzvs4rzxkGwMVGJFZemoksNqf2EP4BnYBcMX9Lwokq\",\"Addrs\":[\"/ip4/10.151.21.17/tcp/5000\"]}" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.824013 6408 server.go:219] "Successfully retrieved NodeIPs" NodeIPs=["10.151.21.17"] Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.824064 6408 server.go:256] "Kube-proxy configuration may be incomplete or incorrect" err="nodePortAddresses is unset; NodePort connections will be accepted on all local IPs. Consider using `--nodeport-addresses primary`" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.827949 6408 server.go:265] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.827978 6408 server_linux.go:132] "Using iptables Proxier" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.831305 6408 proxier.go:242] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.832739 6408 proxier.go:270] "Failed to create nfacct runner, nfacct based metrics won't be available" err="nfacct sub-system not available" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:35:59.834078 6408 proxier.go:270] "Failed to create nfacct runner, nfacct based metrics won't be available" err="nfacct sub-system not available" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.834095 6408 server.go:527] "Version info" version="v1.34.4+k3s1" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.834100 6408 server.go:529] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK="" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836791 6408 config.go:200] "Starting service config controller" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836800 6408 shared_informer.go:349] "Waiting for caches to sync" controller="service config" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836837 6408 config.go:403] "Starting serviceCIDR config controller" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836847 6408 shared_informer.go:349] "Waiting for caches to sync" controller="serviceCIDR config" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836862 6408 config.go:106] "Starting endpoint slice config controller" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836865 6408 shared_informer.go:349] "Waiting for caches to sync" controller="endpoint slice config" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836863 6408 config.go:309] "Starting node config controller" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.836872 6408 shared_informer.go:349] "Waiting for caches to sync" controller="node config" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.937497 6408 shared_informer.go:356] "Caches are synced" controller="service config" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.937612 6408 shared_informer.go:356] "Caches are synced" controller="endpoint slice config" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.937744 6408 shared_informer.go:356] "Caches are synced" controller="serviceCIDR config" Mar 13 10:35:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:35:59.937763 6408 shared_informer.go:356] "Caches are synced" controller="node config" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.012165 6408 apiserver.go:52] "Watching apiserver" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.114910 6408 desired_state_of_world_populator.go:154] "Finished populating initial desired state of world" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128101 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"tmp-cc-detector-mlservice\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-cc-detector-mlservice\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128119 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/844f5823-c427-49b7-9644-0d34195fb121-trust-bundle\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128131 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"state-dir\" (UniqueName: \"kubernetes.io/empty-dir/1b29cd3b-e8c7-4ca0-9209-453435adde6b-state-dir\") pod \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\" (UID: \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128140 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"kube-api-access-9btf5\" (UniqueName: \"kubernetes.io/projected/1b29cd3b-e8c7-4ca0-9209-453435adde6b-kube-api-access-9btf5\") pod \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\" (UID: \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128149 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"tmp-download-worker\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-download-worker\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128158 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-certificates\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128166 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"kube-api-access-kn7mm\" (UniqueName: \"kubernetes.io/projected/844f5823-c427-49b7-9644-0d34195fb121-kube-api-access-kn7mm\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128193 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/1b29cd3b-e8c7-4ca0-9209-453435adde6b-shared-certificates\") pod \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\" (UID: \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128204 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/1b29cd3b-e8c7-4ca0-9209-453435adde6b-trust-bundle\") pod \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\" (UID: \"1b29cd3b-e8c7-4ca0-9209-453435adde6b\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128214 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"config\" (UniqueName: \"kubernetes.io/configmap/844f5823-c427-49b7-9644-0d34195fb121-config\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128223 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"detector-config\" (UniqueName: \"kubernetes.io/configmap/844f5823-c427-49b7-9644-0d34195fb121-detector-config\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128232 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"shared-data\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-data\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128242 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"tmp-cc-detector\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-cc-detector\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128254 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"shared-socket-directory\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-socket-directory\") pod \"844f5823-c427-49b7-9644-0d34195fb121\" (UID: \"844f5823-c427-49b7-9644-0d34195fb121\") " Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128278 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"dnld-storage\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-dnld-storage\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128288 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"installation-storage\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-installation-storage\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128298 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"rancher\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-rancher\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128309 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"crictl\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-crictl\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128318 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/4f679831-c7be-46a9-bab6-ea408cd29c72-trust-bundle\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128317 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1b29cd3b-e8c7-4ca0-9209-453435adde6b-state-dir" (OuterVolumeSpecName: "state-dir") pod "1b29cd3b-e8c7-4ca0-9209-453435adde6b" (UID: "1b29cd3b-e8c7-4ca0-9209-453435adde6b"). InnerVolumeSpecName "state-dir". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128586 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-cc-detector-mlservice" (OuterVolumeSpecName: "tmp-cc-detector-mlservice") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "tmp-cc-detector-mlservice". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128708 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-socket-directory" (OuterVolumeSpecName: "shared-socket-directory") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "shared-socket-directory". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128743 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-certificates" (OuterVolumeSpecName: "shared-certificates") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "shared-certificates". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128870 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-download-worker" (OuterVolumeSpecName: "tmp-download-worker") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "tmp-download-worker". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128964 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/1b29cd3b-e8c7-4ca0-9209-453435adde6b-shared-certificates" (OuterVolumeSpecName: "shared-certificates") pod "1b29cd3b-e8c7-4ca0-9209-453435adde6b" (UID: "1b29cd3b-e8c7-4ca0-9209-453435adde6b"). InnerVolumeSpecName "shared-certificates". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.128972 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"training-samples-storage\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-training-samples-storage\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129044 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"monitoring-log\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-monitoring-log\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129074 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/4f679831-c7be-46a9-bab6-ea408cd29c72-shared-certificates\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129100 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-rfrl4\" (UniqueName: \"kubernetes.io/projected/4f679831-c7be-46a9-bab6-ea408cd29c72-kube-api-access-rfrl4\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129117 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"sys\" (UniqueName: \"kubernetes.io/host-path/48d05fd1-9744-41fd-83cb-f32cb8b2ed30-sys\") pod \"priv-appliance-prometheus-node-exporter-qtwlg\" (UID: \"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\") " pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129139 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/configmap/4f679831-c7be-46a9-bab6-ea408cd29c72-config\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129115 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-data" (OuterVolumeSpecName: "shared-data") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "shared-data". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129153 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"varlog\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-varlog\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129169 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"proc\" (UniqueName: \"kubernetes.io/host-path/48d05fd1-9744-41fd-83cb-f32cb8b2ed30-proc\") pod \"priv-appliance-prometheus-node-exporter-qtwlg\" (UID: \"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\") " pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129234 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"containerdsocket\" (UniqueName: \"kubernetes.io/host-path/4f679831-c7be-46a9-bab6-ea408cd29c72-containerdsocket\") pod \"priv-appliance-monitor-status-zm4mj\" (UID: \"4f679831-c7be-46a9-bab6-ea408cd29c72\") " pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129263 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"root\" (UniqueName: \"kubernetes.io/host-path/48d05fd1-9744-41fd-83cb-f32cb8b2ed30-root\") pod \"priv-appliance-prometheus-node-exporter-qtwlg\" (UID: \"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\") " pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129300 6408 reconciler_common.go:299] "Volume detached for volume \"tmp-download-worker\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-download-worker\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129316 6408 reconciler_common.go:299] "Volume detached for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/1b29cd3b-e8c7-4ca0-9209-453435adde6b-shared-certificates\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129326 6408 reconciler_common.go:299] "Volume detached for volume \"shared-socket-directory\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-socket-directory\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129336 6408 reconciler_common.go:299] "Volume detached for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-certificates\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129348 6408 reconciler_common.go:299] "Volume detached for volume \"shared-data\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-shared-data\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129355 6408 reconciler_common.go:299] "Volume detached for volume \"tmp-cc-detector-mlservice\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-cc-detector-mlservice\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129361 6408 reconciler_common.go:299] "Volume detached for volume \"state-dir\" (UniqueName: \"kubernetes.io/empty-dir/1b29cd3b-e8c7-4ca0-9209-453435adde6b-state-dir\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129389 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/1b29cd3b-e8c7-4ca0-9209-453435adde6b-kube-api-access-9btf5" (OuterVolumeSpecName: "kube-api-access-9btf5") pod "1b29cd3b-e8c7-4ca0-9209-453435adde6b" (UID: "1b29cd3b-e8c7-4ca0-9209-453435adde6b"). InnerVolumeSpecName "kube-api-access-9btf5". PluginName "kubernetes.io/projected", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129494 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/844f5823-c427-49b7-9644-0d34195fb121-kube-api-access-kn7mm" (OuterVolumeSpecName: "kube-api-access-kn7mm") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "kube-api-access-kn7mm". PluginName "kubernetes.io/projected", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129585 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/1b29cd3b-e8c7-4ca0-9209-453435adde6b-trust-bundle" (OuterVolumeSpecName: "trust-bundle") pod "1b29cd3b-e8c7-4ca0-9209-453435adde6b" (UID: "1b29cd3b-e8c7-4ca0-9209-453435adde6b"). InnerVolumeSpecName "trust-bundle". PluginName "kubernetes.io/projected", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129640 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/configmap/844f5823-c427-49b7-9644-0d34195fb121-config" (OuterVolumeSpecName: "config") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "config". PluginName "kubernetes.io/configmap", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.129840 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-cc-detector" (OuterVolumeSpecName: "tmp-cc-detector") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "tmp-cc-detector". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.131127 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/844f5823-c427-49b7-9644-0d34195fb121-trust-bundle" (OuterVolumeSpecName: "trust-bundle") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "trust-bundle". PluginName "kubernetes.io/projected", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.131756 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/configmap/844f5823-c427-49b7-9644-0d34195fb121-detector-config" (OuterVolumeSpecName: "detector-config") pod "844f5823-c427-49b7-9644-0d34195fb121" (UID: "844f5823-c427-49b7-9644-0d34195fb121"). InnerVolumeSpecName "detector-config". PluginName "kubernetes.io/configmap", VolumeGIDValue "" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.160896 6408 kube.go:163] Node controller sync successful Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.160947 6408 vxlan.go:128] VXLAN config: VNI=1 Port=0 GBP=false Learning=false DirectRouting=false Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.163962 6408 kube.go:704] List of node(ip-10-151-21-17.us-west-2.compute.internal) annotations: map[string]string{"k3s.io/node-args":"[\"agent\",\"--server\",\"https://10.151.22.85:6443\",\"--token-file\",\"/etc/rancher/k3s/securiti-token\",\"--data-dir\",\"/mnt/rancher/k3s\",\"--selinux\",\"false\",\"--disable-default-registry-endpoint\",\"true\",\"--debug\",\"true\",\"--node-label\",\"role=worker\",\"--node-label\",\"node-role.appliance.securiti.io/worker=true\",\"--kubelet-arg\",\"config=/etc/rancher/k3s/kubelet/config.yaml\"]", "k3s.io/node-config-hash":"UCASGTXDKIEAFOS2WRU4BQTXT55QYQTQ5I2FJ4CPYDDLWBVXQAKQ====", "k3s.io/node-env":"{\"K3S_P2P_PORT\":\"5000\"}", "node.alpha.kubernetes.io/ttl":"0", "p2p.k3s.cattle.io/node-address":"/ip4/10.151.21.17/tcp/5000/p2p/QmXbGzvs4rzxkGwMVGJFZemoksNqf2EP4BnYBcMX9Lwokq", "p2p.k3s.cattle.io/node-addresses":"{\"ID\":\"QmXbGzvs4rzxkGwMVGJFZemoksNqf2EP4BnYBcMX9Lwokq\",\"Addrs\":[\"/ip4/10.151.21.17/tcp/5000\"]}", "volumes.kubernetes.io/controller-managed-attach-detach":"true"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.168989 6408 iptables.go:50] Starting flannel in iptables mode... Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=warning msg="no subnet found for key: FLANNEL_NETWORK in file: /run/flannel/subnet.env" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=warning msg="no subnet found for key: FLANNEL_SUBNET in file: /run/flannel/subnet.env" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=warning msg="no subnet found for key: FLANNEL_IPV6_NETWORK in file: /run/flannel/subnet.env" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=warning msg="no subnet found for key: FLANNEL_IPV6_SUBNET in file: /run/flannel/subnet.env" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.169039 6408 iptables.go:101] Current network or subnet (10.42.0.0/16, 10.42.1.0/24) is not equal to previous one (0.0.0.0/0, 0.0.0.0/0), trying to recycle old iptables rules Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.169278 6408 kube.go:558] Creating the node lease for IPv4. This is the n.Spec.PodCIDRs: [10.42.1.0/24] Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.177982 6408 iptables.go:111] Setting up masking rules Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.180314 6408 iptables.go:212] Changing default FORWARD chain policy to ACCEPT Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=info msg="Wrote flannel subnet file to /run/flannel/subnet.env" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=info msg="Running flannel backend." Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.181834 6408 vxlan_network.go:68] watching for new subnet leases Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.181854 6408 vxlan_network.go:115] starting vxlan device watcher Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.181858 6408 subnet.go:150] Batch elem [0] is { lease.Event{Type:0, Lease:lease.Lease{EnableIPv4:true, EnableIPv6:false, Subnet:ip.IP4Net{IP:0xa2a0000, PrefixLen:0x18}, IPv6Subnet:ip.IP6Net{IP:(*ip.IP6)(nil), PrefixLen:0x0}, Attrs:lease.LeaseAttrs{PublicIP:0xa971655, PublicIPv6:(*ip.IP6)(nil), BackendType:"vxlan", BackendData:json.RawMessage{0x7b, 0x22, 0x56, 0x4e, 0x49, 0x22, 0x3a, 0x31, 0x2c, 0x22, 0x56, 0x74, 0x65, 0x70, 0x4d, 0x41, 0x43, 0x22, 0x3a, 0x22, 0x34, 0x61, 0x3a, 0x62, 0x31, 0x3a, 0x65, 0x64, 0x3a, 0x39, 0x31, 0x3a, 0x38, 0x66, 0x3a, 0x32, 0x65, 0x22, 0x7d}, BackendV6Data:json.RawMessage(nil)}, Expiration:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), Asof:0}} } Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.181905 6408 vxlan_network.go:265] Received Subnet Event with VxLan: BackendType: vxlan, PublicIP: 10.151.22.85, PublicIPv6: (nil), BackendData: {"VNI":1,"VtepMAC":"4a:b1:ed:91:8f:2e"}, BackendV6Data: (nil) Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.189566 6408 iptables.go:358] bootstrap done Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.196072 6408 iptables.go:358] bootstrap done Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230311 6408 reconciler_common.go:299] "Volume detached for volume \"kube-api-access-9btf5\" (UniqueName: \"kubernetes.io/projected/1b29cd3b-e8c7-4ca0-9209-453435adde6b-kube-api-access-9btf5\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230332 6408 reconciler_common.go:299] "Volume detached for volume \"kube-api-access-kn7mm\" (UniqueName: \"kubernetes.io/projected/844f5823-c427-49b7-9644-0d34195fb121-kube-api-access-kn7mm\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230338 6408 reconciler_common.go:299] "Volume detached for volume \"detector-config\" (UniqueName: \"kubernetes.io/configmap/844f5823-c427-49b7-9644-0d34195fb121-detector-config\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230344 6408 reconciler_common.go:299] "Volume detached for volume \"tmp-cc-detector\" (UniqueName: \"kubernetes.io/empty-dir/844f5823-c427-49b7-9644-0d34195fb121-tmp-cc-detector\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230349 6408 reconciler_common.go:299] "Volume detached for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/1b29cd3b-e8c7-4ca0-9209-453435adde6b-trust-bundle\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230355 6408 reconciler_common.go:299] "Volume detached for volume \"config\" (UniqueName: \"kubernetes.io/configmap/844f5823-c427-49b7-9644-0d34195fb121-config\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230360 6408 reconciler_common.go:299] "Volume detached for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/844f5823-c427-49b7-9644-0d34195fb121-trust-bundle\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:00.230708 6408 swap_util.go:74] "error creating dir to test if tmpfs noswap is enabled. Assuming not supported" mount path="" error="stat /var/lib/kubelet/plugins/kubernetes.io/empty-dir: no such file or directory" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.323998623Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.324089551Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.324113562Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.326946377Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.327082092Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.327109506Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.343062462Z" level=debug msg="begin cni setup" podsandboxid=c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.345191502Z" level=debug msg="begin cni setup" podsandboxid=88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:00.364265228Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.2\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"12:35:9c:8f:19:11\",\"Sandbox\":\"/var/run/netns/cni-6fc443b7-28ca-425f-6311-ac1c43d1adf1\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-6fc443b7-28ca-425f-6311-ac1c43d1adf1\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth1c338d9f\":{\"IPConfigs\":null,\"Mac\":\"d6:f4:4b:fa:52:c3\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.366210411Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.3\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"f2:18:3d:fd:a4:48\",\"Sandbox\":\"/var/run/netns/cni-8bcdd624-fec5-86f6-72b2-8575f1f4e155\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-8bcdd624-fec5-86f6-72b2-8575f1f4e155\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth5b97af7f\":{\"IPConfigs\":null,\"Mac\":\"06:91:0d:32:fb:c9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.366589386Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.366622084Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.368686548Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.368708916Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.370462030Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.372493611Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.382259160Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.382279422Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.382520577Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.382538411Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.384Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.385Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.385Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.385Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.385Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.385Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=info handler=mirror ip=127.0.0.1 latency=5.665215ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=info handler=mirror ip=127.0.0.1 latency=5.114891ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.389955077Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:00 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.389987553Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:00 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.390017137Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.390059586Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.389991193Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.390102995Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.390966271Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.399046399Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.399071282Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.401Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.402Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.402Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.402Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.30081ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.402520245Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:00 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.402595398Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.404584615Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.406761931Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.408796869Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.408823279Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.410Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.411Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.411Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:00.411Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.320176ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.411766033Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:00 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.411838954Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.416653445Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.439614295Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.439826 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.439865 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.439880 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.439920 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"88dd4676e9658c2cd7fcabd46bfd21433231988c1ab886751b72586f83d79bef\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.455765545Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.455873 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.455899 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.455912 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:00.455945 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"c4bceb90773c35eb471e773a03598a1929ea137a078f66da91952045583c41ea\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.621033787Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.623622339Z" level=debug msg="schedule content cleanup" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00.623655154Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:00 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:00Z" level=debug msg="Settled apiserver addresses sync: [10.151.22.85:6443]" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.028526 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="1b29cd3b-e8c7-4ca0-9209-453435adde6b" path="/var/lib/kubelet/pods/1b29cd3b-e8c7-4ca0-9209-453435adde6b/volumes" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.028667 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="4e6f3406-a713-45d8-b2a0-c9da15debc2a" path="/var/lib/kubelet/pods/4e6f3406-a713-45d8-b2a0-c9da15debc2a/volumes" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.030070 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="844f5823-c427-49b7-9644-0d34195fb121" path="/var/lib/kubelet/pods/844f5823-c427-49b7-9644-0d34195fb121/volumes" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.030657 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="bb8f5084-d171-4c7b-9c1b-356ef770a281" path="/var/lib/kubelet/pods/bb8f5084-d171-4c7b-9c1b-356ef770a281/volumes" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.030861 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="beb7b1e5-8adc-41e0-b2fb-c3487a987cb0" path="/var/lib/kubelet/pods/beb7b1e5-8adc-41e0-b2fb-c3487a987cb0/volumes" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.031041 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="e44c43e3-b17a-4df4-a818-df58119752dc" path="/var/lib/kubelet/pods/e44c43e3-b17a-4df4-a818-df58119752dc/volumes" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.031453 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="ed8b7ef4-1210-4ea8-9d0a-222c8fbfd6a1" path="/var/lib/kubelet/pods/ed8b7ef4-1210-4ea8-9d0a-222c8fbfd6a1/volumes" Mar 13 10:36:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:01.031644 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="f5e5f7c7-bc8f-43c9-86a1-6c5f403b97b0" path="/var/lib/kubelet/pods/f5e5f7c7-bc8f-43c9-86a1-6c5f403b97b0/volumes" Mar 13 10:36:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:06.031Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw=="} Mar 13 10:36:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:06.031Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000087147} Mar 13 10:36:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:06.031Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000106582} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.131Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw=="} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.131Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.00006917} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.132Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000081661} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.132Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiDb5qTiEasC06/p6elIpHdxQ+Bnm53RXJdZqL5i+dpWTg=="} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.132Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiDb5qTiEasC06/p6elIpHdxQ+Bnm53RXJdZqL5i+dpWTg==", "time": 0.00006033} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.132Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiDb5qTiEasC06/p6elIpHdxQ+Bnm53RXJdZqL5i+dpWTg==", "time": 0.000073538} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.133Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiCwZyrC46gE88LKIEUMKay7SzZ1p+juymZLzRKm9I7RJA=="} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.133Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiCwZyrC46gE88LKIEUMKay7SzZ1p+juymZLzRKm9I7RJA==", "time": 0.000041104} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.133Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiCwZyrC46gE88LKIEUMKay7SzZ1p+juymZLzRKm9I7RJA==", "time": 0.000050095} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.134Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAOqY/lZLrGI35PphW0wuPk29H7XYHruI4pL5B5bYNouA=="} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.134Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAOqY/lZLrGI35PphW0wuPk29H7XYHruI4pL5B5bYNouA==", "time": 0.000046224} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.134Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAOqY/lZLrGI35PphW0wuPk29H7XYHruI4pL5B5bYNouA==", "time": 0.000057725} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.134Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiBr5gUWF3LhQ5gyz3SOVhw/0iAQIUvppksGTOlnVtjOWQ=="} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.134Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiBr5gUWF3LhQ5gyz3SOVhw/0iAQIUvppksGTOlnVtjOWQ==", "time": 0.000052335} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.134Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiBr5gUWF3LhQ5gyz3SOVhw/0iAQIUvppksGTOlnVtjOWQ==", "time": 0.000063483} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.135Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAHT7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.135Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAHT7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000045929} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.135Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAHT7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000057763} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.136Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAACRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.136Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAACRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000048129} Mar 13 10:36:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:07.136Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAACRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000059815} Mar 13 10:36:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.035415037Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.035526620Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.035553282Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.042861565Z" level=debug msg="begin cni setup" podsandboxid=8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc00011c650), "name":"cbr0", "type":"bridge"} Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:12.060370939Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.4\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ae:ab:19:df:b8:18\",\"Sandbox\":\"/var/run/netns/cni-296b4137-15ff-5ab7-fef6-b1b3e9283c6f\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-296b4137-15ff-5ab7-fef6-b1b3e9283c6f\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth892748ce\":{\"IPConfigs\":null,\"Mac\":\"b2:2f:a7:84:3e:6d\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.062725115Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.062747443Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.064815419Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.065037353Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.065049231Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12Z" level=info handler=mirror ip=127.0.0.1 latency=3.342768ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.070478527Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:12 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.070509701Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.070539767Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.070769291Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.072899604Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.072917150Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.074Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.075Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.075Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:12.075Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.02191ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.075485135Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:12 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.075562820Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.079648640Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.115272219Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:12.115424 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:12.115460 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:12.115487 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:12.115539 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"8710b63cd731e301e7ecc07d20d205b3a76c600b29e414ca046bd191e4e566a7\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.162536438Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.164414094Z" level=debug msg="schedule content cleanup" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12.164432673Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:12Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.029446854Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.029550869Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.029575654Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.036867656Z" level=debug msg="begin cni setup" podsandboxid=a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:14.053880582Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.5\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"2e:bd:2f:6a:6f:0e\",\"Sandbox\":\"/var/run/netns/cni-6acc42f7-82c8-1716-f57d-a0f59aab3eeb\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-6acc42f7-82c8-1716-f57d-a0f59aab3eeb\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth9e0dc771\":{\"IPConfigs\":null,\"Mac\":\"a2:5a:ad:a2:6e:a4\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.056118871Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.056141903Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.058277600Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.058491093Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.058502743Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14Z" level=info handler=mirror ip=127.0.0.1 latency=2.124204ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.062330880Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:14 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.062351974Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.062375806Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.062596424Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.064921148Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.064953053Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:14.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="899.475µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.068276049Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:14 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.068345072Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.072753059Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.101239643Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:14.101356 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:14.101390 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:14.101422 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:14.101460 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"a62f0033790b8562fb1d9334cb1d2dc90137753a145b82bbf856a570cd1f52d3\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.128484407Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.130700826Z" level=debug msg="schedule content cleanup" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14.130732507Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:14Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.029365509Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.029501814Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.029521696Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.037304128Z" level=debug msg="begin cni setup" podsandboxid=81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:26.055719773Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.6\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"7a:c4:4e:8d:84:9c\",\"Sandbox\":\"/var/run/netns/cni-1a29c289-633b-9c06-bfa9-4546d99c6be6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-1a29c289-633b-9c06-bfa9-4546d99c6be6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth89a6ca5c\":{\"IPConfigs\":null,\"Mac\":\"92:8b:24:8a:d8:cd\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.057855407Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.057876881Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.059860070Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.060075864Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.060089488Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26Z" level=info handler=mirror ip=127.0.0.1 latency=2.896072ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.064844617Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:26 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.064865930Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.064892190Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.065087177Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.067278620Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.067295459Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:26.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.102582ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.070088502Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:26 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.070153274Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.074121303Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.119178781Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:26.119298 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:26.119329 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:26.119346 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:26.119385 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"81a772859fdff0311a974f6bcab497ddd1e83ff48d6e46884875e73bfd5bc3e0\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.160446019Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.162633396Z" level=debug msg="schedule content cleanup" Mar 13 10:36:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:26.162663631Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.030153159Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.030264758Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.030288864Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.037444247Z" level=debug msg="begin cni setup" podsandboxid=351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:28.053853779Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.7\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"2e:59:ff:26:ec:8d\",\"Sandbox\":\"/var/run/netns/cni-dfdd2c85-a7b9-1801-159e-92520770efbf\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-dfdd2c85-a7b9-1801-159e-92520770efbf\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethf813d230\":{\"IPConfigs\":null,\"Mac\":\"fa:c5:aa:36:52:3c\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.056053355Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.056071417Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.057909294Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.058106083Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.058120163Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28Z" level=info handler=mirror ip=127.0.0.1 latency=1.528103ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.061320408Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:28 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.061345863Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.061373152Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.061600378Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.063673251Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.063691432Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:28.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="830.963µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.066131674Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:28 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.066171824Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.070023400Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.106274707Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:28.106383 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:28.106434 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:28.106449 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:28.106485 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"351453387e747f49d4cc470191db93fb08feae37901c40c2fd7be2b974444ace\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.126469464Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.128897405Z" level=debug msg="schedule content cleanup" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28.128928932Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:28Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:30Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:30Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.029788270Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.029893903Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.029915221Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.037736511Z" level=debug msg="begin cni setup" podsandboxid=2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:41.055835165Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.8\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"b2:0b:63:56:0f:70\",\"Sandbox\":\"/var/run/netns/cni-16e4b3c2-6cd9-7597-2bab-c6ec39bb221b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-16e4b3c2-6cd9-7597-2bab-c6ec39bb221b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth833b8269\":{\"IPConfigs\":null,\"Mac\":\"26:fc:e1:5b:cb:d6\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.057887814Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.057909543Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.059819618Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.060038112Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.060050879Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41Z" level=info handler=mirror ip=127.0.0.1 latency=2.938013ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.064925488Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:41 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.064951487Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.064980036Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.065198249Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.067195824Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.067212997Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:41.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.02268ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.070025578Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:41 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.070076629Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.074055568Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.104369658Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:41.104510 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:41.104543 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:41.104560 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:41.104601 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"2684b9eafd28151aea279362c1887c50f124887e06de535263ca034fdfc951d1\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.140670797Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.142778079Z" level=debug msg="schedule content cleanup" Mar 13 10:36:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:41.142818272Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.030170196Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.030280348Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.030302805Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.037411205Z" level=debug msg="begin cni setup" podsandboxid=94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:42.054451936Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.9\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ce:6f:39:99:50:3e\",\"Sandbox\":\"/var/run/netns/cni-a6cbc99b-00a8-4726-00ac-8ce7d06ee874\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-a6cbc99b-00a8-4726-00ac-8ce7d06ee874\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth11631126\":{\"IPConfigs\":null,\"Mac\":\"16:50:71:2e:33:e6\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.056701852Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.056721700Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.058591734Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.058772865Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.058787508Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42Z" level=info handler=mirror ip=127.0.0.1 latency=2.164686ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.062603500Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:42 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.062620952Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.062636197Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.062814397Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.064918065Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.064936272Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:42.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.0612ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.067740075Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:42 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.067789339Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.071980921Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.105387783Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:42.105512 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:42.105542 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:42.105557 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:42.105595 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"94288dad82e371ce79e70848cc83669e2e93f864015ad898301aa13d5e5afd73\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.124631649Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.126955981Z" level=debug msg="schedule content cleanup" Mar 13 10:36:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:42.126985583Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:44 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:44Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:44 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:44Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:46Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:46Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.029500609Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.029616826Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.029639828Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.036688962Z" level=debug msg="begin cni setup" podsandboxid=14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000106650), "name":"cbr0", "type":"bridge"} Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:56.053845687Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.10\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"02:84:6d:3a:95:54\",\"Sandbox\":\"/var/run/netns/cni-36828ee9-a42c-0810-dff9-0a2d185112c9\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-36828ee9-a42c-0810-dff9-0a2d185112c9\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth3834a1dc\":{\"IPConfigs\":null,\"Mac\":\"ce:3a:83:f6:f8:99\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.056137795Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.056159772Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.058132751Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.058391375Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.058428102Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56Z" level=info handler=mirror ip=127.0.0.1 latency=3.297093ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.063644280Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:56 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.063675075Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.063727461Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.063961107Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.066168369Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.066190353Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:56.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.036164ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.068926692Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:56 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.068969972Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.073791861Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.105208930Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:56.105318 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:56.105349 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:56.105363 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:56.105419 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"14becd4895001c3d44139f6dab91211840afe69c0e82ec958e208b803dc2871a\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.120486324Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.122542992Z" level=debug msg="schedule content cleanup" Mar 13 10:36:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:56.122563770Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.029426043Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.029525999Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.029550063Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.035881195Z" level=debug msg="begin cni setup" podsandboxid=611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:36:57.053333465Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.11\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"2e:95:5c:b5:be:e3\",\"Sandbox\":\"/var/run/netns/cni-f2b1d209-13ed-05e4-2bb1-b86d631c26c9\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-f2b1d209-13ed-05e4-2bb1-b86d631c26c9\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth388c7e4f\":{\"IPConfigs\":null,\"Mac\":\"26:59:e0:46:f4:b5\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.055550352Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.055568317Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.057572208Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.057763836Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.057778259Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57Z" level=info handler=mirror ip=127.0.0.1 latency=2.345914ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.061805811Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:36:57 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.061828157Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.061851071Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.062101498Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.064029231Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.064050010Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:36:57.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.037882ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.066684993Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:36:57 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.066758081Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.070928168Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.104906636Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.106894135Z" level=debug msg="schedule content cleanup" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.106986429Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:57.110815551Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:57.111044 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:57.111076 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:57.111089 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:36:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:57.111126 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"611b1a5dbd29aeb90d9dc30f45712a500c47292e3bfe83f599008252530d7883\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.016630598Z" level=error msg="ContainerStatus for \"5e3ec5d2dedfba3fac5e4625daa9a7680db710a2581b949d606095f308cd3e62\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"5e3ec5d2dedfba3fac5e4625daa9a7680db710a2581b949d606095f308cd3e62\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.016731 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"5e3ec5d2dedfba3fac5e4625daa9a7680db710a2581b949d606095f308cd3e62\": not found" containerID="5e3ec5d2dedfba3fac5e4625daa9a7680db710a2581b949d606095f308cd3e62" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.016755 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="5e3ec5d2dedfba3fac5e4625daa9a7680db710a2581b949d606095f308cd3e62" err="rpc error: code = NotFound desc = an error occurred when try to find container \"5e3ec5d2dedfba3fac5e4625daa9a7680db710a2581b949d606095f308cd3e62\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.016892174Z" level=error msg="ContainerStatus for \"f864b8d131f215badba83432d6b1ba538bc6fd0225b9d07ecefc5b39bf6220e9\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"f864b8d131f215badba83432d6b1ba538bc6fd0225b9d07ecefc5b39bf6220e9\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.017148 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"f864b8d131f215badba83432d6b1ba538bc6fd0225b9d07ecefc5b39bf6220e9\": not found" containerID="f864b8d131f215badba83432d6b1ba538bc6fd0225b9d07ecefc5b39bf6220e9" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.017177 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="f864b8d131f215badba83432d6b1ba538bc6fd0225b9d07ecefc5b39bf6220e9" err="rpc error: code = NotFound desc = an error occurred when try to find container \"f864b8d131f215badba83432d6b1ba538bc6fd0225b9d07ecefc5b39bf6220e9\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.017383033Z" level=error msg="ContainerStatus for \"f742c69c6f5c0c99f641d2a7b6baa793c90f14713663f3adf1d4921e85c5be1e\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"f742c69c6f5c0c99f641d2a7b6baa793c90f14713663f3adf1d4921e85c5be1e\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.017488 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"f742c69c6f5c0c99f641d2a7b6baa793c90f14713663f3adf1d4921e85c5be1e\": not found" containerID="f742c69c6f5c0c99f641d2a7b6baa793c90f14713663f3adf1d4921e85c5be1e" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.017501 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="f742c69c6f5c0c99f641d2a7b6baa793c90f14713663f3adf1d4921e85c5be1e" err="rpc error: code = NotFound desc = an error occurred when try to find container \"f742c69c6f5c0c99f641d2a7b6baa793c90f14713663f3adf1d4921e85c5be1e\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.017829006Z" level=error msg="ContainerStatus for \"1c68cd770866dfef4dfbe12fa1f409248ca6544175c6719b1c363189c6a9d1a8\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"1c68cd770866dfef4dfbe12fa1f409248ca6544175c6719b1c363189c6a9d1a8\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.017896 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"1c68cd770866dfef4dfbe12fa1f409248ca6544175c6719b1c363189c6a9d1a8\": not found" containerID="1c68cd770866dfef4dfbe12fa1f409248ca6544175c6719b1c363189c6a9d1a8" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.017911 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="1c68cd770866dfef4dfbe12fa1f409248ca6544175c6719b1c363189c6a9d1a8" err="rpc error: code = NotFound desc = an error occurred when try to find container \"1c68cd770866dfef4dfbe12fa1f409248ca6544175c6719b1c363189c6a9d1a8\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.018087511Z" level=error msg="ContainerStatus for \"ff64fdb2590952ebf6891f650d6f908568500292f567c9c2fd52e1136f4cc86f\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"ff64fdb2590952ebf6891f650d6f908568500292f567c9c2fd52e1136f4cc86f\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.018177 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"ff64fdb2590952ebf6891f650d6f908568500292f567c9c2fd52e1136f4cc86f\": not found" containerID="ff64fdb2590952ebf6891f650d6f908568500292f567c9c2fd52e1136f4cc86f" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.018188 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="ff64fdb2590952ebf6891f650d6f908568500292f567c9c2fd52e1136f4cc86f" err="rpc error: code = NotFound desc = an error occurred when try to find container \"ff64fdb2590952ebf6891f650d6f908568500292f567c9c2fd52e1136f4cc86f\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.018488149Z" level=error msg="ContainerStatus for \"b1587aa2e2860ef52651eaa864e3a0cf409ed55748902b6fe3f85025d1ef8f0c\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"b1587aa2e2860ef52651eaa864e3a0cf409ed55748902b6fe3f85025d1ef8f0c\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.018555 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"b1587aa2e2860ef52651eaa864e3a0cf409ed55748902b6fe3f85025d1ef8f0c\": not found" containerID="b1587aa2e2860ef52651eaa864e3a0cf409ed55748902b6fe3f85025d1ef8f0c" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.018566 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="b1587aa2e2860ef52651eaa864e3a0cf409ed55748902b6fe3f85025d1ef8f0c" err="rpc error: code = NotFound desc = an error occurred when try to find container \"b1587aa2e2860ef52651eaa864e3a0cf409ed55748902b6fe3f85025d1ef8f0c\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.018863686Z" level=error msg="ContainerStatus for \"4573f3aacbc0c6f9cd84d4e479e2ea3ddbb0b41f0ef1e7229b6b9af9b7576e7b\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"4573f3aacbc0c6f9cd84d4e479e2ea3ddbb0b41f0ef1e7229b6b9af9b7576e7b\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.018950 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"4573f3aacbc0c6f9cd84d4e479e2ea3ddbb0b41f0ef1e7229b6b9af9b7576e7b\": not found" containerID="4573f3aacbc0c6f9cd84d4e479e2ea3ddbb0b41f0ef1e7229b6b9af9b7576e7b" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.018969 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="4573f3aacbc0c6f9cd84d4e479e2ea3ddbb0b41f0ef1e7229b6b9af9b7576e7b" err="rpc error: code = NotFound desc = an error occurred when try to find container \"4573f3aacbc0c6f9cd84d4e479e2ea3ddbb0b41f0ef1e7229b6b9af9b7576e7b\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.019127991Z" level=error msg="ContainerStatus for \"54bbe95e8f209851612d783782205386aaa2d03602186ac9d6e42aa631846a34\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"54bbe95e8f209851612d783782205386aaa2d03602186ac9d6e42aa631846a34\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.019200 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"54bbe95e8f209851612d783782205386aaa2d03602186ac9d6e42aa631846a34\": not found" containerID="54bbe95e8f209851612d783782205386aaa2d03602186ac9d6e42aa631846a34" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.019210 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="54bbe95e8f209851612d783782205386aaa2d03602186ac9d6e42aa631846a34" err="rpc error: code = NotFound desc = an error occurred when try to find container \"54bbe95e8f209851612d783782205386aaa2d03602186ac9d6e42aa631846a34\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.019470809Z" level=error msg="ContainerStatus for \"5df2b8f66df90210cd502287d35482d66572a9d8971531872140b50722071391\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"5df2b8f66df90210cd502287d35482d66572a9d8971531872140b50722071391\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.019558 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"5df2b8f66df90210cd502287d35482d66572a9d8971531872140b50722071391\": not found" containerID="5df2b8f66df90210cd502287d35482d66572a9d8971531872140b50722071391" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.019571 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="5df2b8f66df90210cd502287d35482d66572a9d8971531872140b50722071391" err="rpc error: code = NotFound desc = an error occurred when try to find container \"5df2b8f66df90210cd502287d35482d66572a9d8971531872140b50722071391\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.019673964Z" level=error msg="ContainerStatus for \"6d47b72c54c4b3012c2ba53622d15a0d48db4e7f583ac47a6f3221e80cfe6c54\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"6d47b72c54c4b3012c2ba53622d15a0d48db4e7f583ac47a6f3221e80cfe6c54\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.019713 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"6d47b72c54c4b3012c2ba53622d15a0d48db4e7f583ac47a6f3221e80cfe6c54\": not found" containerID="6d47b72c54c4b3012c2ba53622d15a0d48db4e7f583ac47a6f3221e80cfe6c54" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.019725 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="6d47b72c54c4b3012c2ba53622d15a0d48db4e7f583ac47a6f3221e80cfe6c54" err="rpc error: code = NotFound desc = an error occurred when try to find container \"6d47b72c54c4b3012c2ba53622d15a0d48db4e7f583ac47a6f3221e80cfe6c54\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.019793012Z" level=error msg="ContainerStatus for \"24a359fbf740d30ff163ba72a168996abba4f60f8c7c33007891c30cf01f5756\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"24a359fbf740d30ff163ba72a168996abba4f60f8c7c33007891c30cf01f5756\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.019835 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"24a359fbf740d30ff163ba72a168996abba4f60f8c7c33007891c30cf01f5756\": not found" containerID="24a359fbf740d30ff163ba72a168996abba4f60f8c7c33007891c30cf01f5756" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.019845 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="24a359fbf740d30ff163ba72a168996abba4f60f8c7c33007891c30cf01f5756" err="rpc error: code = NotFound desc = an error occurred when try to find container \"24a359fbf740d30ff163ba72a168996abba4f60f8c7c33007891c30cf01f5756\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59.019932927Z" level=error msg="ContainerStatus for \"4353a8cafb6c8529fe702f7ff4a1e46d88f93fe5cf2e5a738be0dd32781f97ca\" failed" error="rpc error: code = NotFound desc = an error occurred when try to find container \"4353a8cafb6c8529fe702f7ff4a1e46d88f93fe5cf2e5a738be0dd32781f97ca\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:36:59.019968 6408 log.go:32] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"4353a8cafb6c8529fe702f7ff4a1e46d88f93fe5cf2e5a738be0dd32781f97ca\": not found" containerID="4353a8cafb6c8529fe702f7ff4a1e46d88f93fe5cf2e5a738be0dd32781f97ca" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:36:59.019977 6408 kuberuntime_gc.go:364] "Error getting ContainerStatus for containerID" containerID="4353a8cafb6c8529fe702f7ff4a1e46d88f93fe5cf2e5a738be0dd32781f97ca" err="rpc error: code = NotFound desc = an error occurred when try to find container \"4353a8cafb6c8529fe702f7ff4a1e46d88f93fe5cf2e5a738be0dd32781f97ca\": not found" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:36:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:36:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:04Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:04Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:06Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:06Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.036260042Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.036376964Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.036418662Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.043382969Z" level=debug msg="begin cni setup" podsandboxid=11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:07.060699831Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.12\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"b2:46:ca:29:7d:20\",\"Sandbox\":\"/var/run/netns/cni-65c10de3-84aa-d011-657b-0ebe5784be80\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-65c10de3-84aa-d011-657b-0ebe5784be80\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vetha3cfdaef\":{\"IPConfigs\":null,\"Mac\":\"72:7c:16:7c:da:6d\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.062884646Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.062906520Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.064952160Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.065196182Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.065208652Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07Z" level=info handler=mirror ip=127.0.0.1 latency=3.259573ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.070420178Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:07 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.070444853Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.070482333Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.071499053Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.074083217Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.074117549Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.076Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.076Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.076Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.076Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="815.462µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.076722238Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:07 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.076810134Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.080716649Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.115840789Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:07.115925 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:07.115955 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:07.115970 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:07.116011 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"11d6896153b558e1e44bca82154f786030c796e0458dee3e69e12a6f18cc3bdd\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:07.136Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:62 error reading message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "error": "stream reset (local): code: 0x0: transport error: stream reset, error code: 0"} Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.174092301Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.176066785Z" level=debug msg="schedule content cleanup" Mar 13 10:37:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:07.176098927Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.031757523Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.031882722Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.031907443Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.038938371Z" level=debug msg="begin cni setup" podsandboxid=45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000184bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:11.055779229Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.13\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"8e:b0:2a:c8:df:2d\",\"Sandbox\":\"/var/run/netns/cni-d68bf051-efd3-2bea-fc15-7f99ca4317de\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-d68bf051-efd3-2bea-fc15-7f99ca4317de\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth6189b46a\":{\"IPConfigs\":null,\"Mac\":\"72:1a:b0:be:d3:83\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.058154818Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.058180897Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.060158878Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.060375656Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.060388338Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11Z" level=info handler=mirror ip=127.0.0.1 latency=2.308708ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.064628346Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:11 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.064653594Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.064676575Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.064916979Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.067589864Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.067614474Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:11.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.025917ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.070283905Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:11 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.070319587Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.074751725Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.102388254Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.104618988Z" level=debug msg="schedule content cleanup" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.104702244Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:11.108348588Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:11.108521 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:11.108563 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:11.108581 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:11 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:11.108621 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"45d5bbcf8742ac90ea6edfa67ac03335e1b4267bd535640e9ecaa0f53c04ef78\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:37:16 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:16Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:16 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:16Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:20Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:20Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.029196234Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.029319554Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.029341353Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.037108392Z" level=debug msg="begin cni setup" podsandboxid=1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:21.054011509Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.14\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"16:5f:8a:1d:d7:cc\",\"Sandbox\":\"/var/run/netns/cni-28ff2316-ef0d-ffa1-3e91-52e6dd0bdbec\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-28ff2316-ef0d-ffa1-3e91-52e6dd0bdbec\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethcf12c560\":{\"IPConfigs\":null,\"Mac\":\"fa:d2:5f:3b:05:cc\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.056277166Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.056302757Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.058327795Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.058567205Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.058581140Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21Z" level=info handler=mirror ip=127.0.0.1 latency=3.026469ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.064638822Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:21 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.064664986Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.064696656Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.064925830Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.067849956Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.067873168Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:21.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.064818ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.070611654Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:21 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.070649697Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.074823747Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.106917127Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:21.107038 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:21.107072 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:21.107087 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:21.107133 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"1e142039316c0fee85243d1b1f005c8ae0e850edc44a37e42be95c8deba68f5e\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.173723436Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.176153495Z" level=debug msg="schedule content cleanup" Mar 13 10:37:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:21.176183315Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw=="} Mar 13 10:37:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.00007558} Mar 13 10:37:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000089753} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.029774293Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.029886585Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.029911901Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.037501272Z" level=debug msg="begin cni setup" podsandboxid=a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:26.053986256Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.15\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"32:3f:cc:f0:a9:ca\",\"Sandbox\":\"/var/run/netns/cni-66504bc2-199b-5a73-3089-afd68de4a214\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-66504bc2-199b-5a73-3089-afd68de4a214\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vetha3856bd4\":{\"IPConfigs\":null,\"Mac\":\"66:91:3f:ad:56:49\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.056085018Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.056105361Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.058127416Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.058363003Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.058374669Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26Z" level=info handler=mirror ip=127.0.0.1 latency=2.407974ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.062669462Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:26 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.062694842Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.062716608Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.062940459Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.064992688Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.065012715Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:26.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="955.001µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.067784413Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:26 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.067836013Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.071851289Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.087289875Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.089380658Z" level=debug msg="schedule content cleanup" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.089416516Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:26.117366773Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:26.117560 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:26.117596 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:26.117617 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:26.117656 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"a7e4f81400af0db5df3c26fbf5cd365793bcbeb48c5f849f917d7dc4b6e5ff7a\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:37:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.029834986Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.029929311Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.029969584Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.037904432Z" level=debug msg="begin cni setup" podsandboxid=e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:32.055540406Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.16\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"12:2f:b7:a3:d0:19\",\"Sandbox\":\"/var/run/netns/cni-ab5da1cd-d726-3f98-033f-b5924ba70632\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-ab5da1cd-d726-3f98-033f-b5924ba70632\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethe768d57e\":{\"IPConfigs\":null,\"Mac\":\"56:18:89:aa:0d:ee\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.057856583Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.057877664Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.059890969Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.060100744Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.060113069Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32Z" level=info handler=mirror ip=127.0.0.1 latency=3.429488ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.065714185Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:32 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.065739722Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.065760427Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.065968697Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.067964476Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.067982224Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:32.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.20587ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.070864160Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:32 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.070914693Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.074931056Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.110431929Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:32.110546 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:32.110575 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:32.110589 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:32.110627 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"e1ee36a217606928b5540a7d97a7e8b4c568eac1b90656fb8217f0ea612e11eb\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.228356396Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.230743675Z" level=debug msg="schedule content cleanup" Mar 13 10:37:32 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:32.230773203Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.029849490Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.029987837Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.030020573Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.037184831Z" level=debug msg="begin cni setup" podsandboxid=df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:40.054425458Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.17\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"3a:fa:31:bf:74:cb\",\"Sandbox\":\"/var/run/netns/cni-279214c8-7a82-3318-90ac-b8dd6a6c5ef9\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-279214c8-7a82-3318-90ac-b8dd6a6c5ef9\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vetha01fe001\":{\"IPConfigs\":null,\"Mac\":\"aa:7f:a8:d9:cd:1b\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.056606290Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.056629546Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.058644524Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.058861702Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.058874100Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40Z" level=info handler=mirror ip=127.0.0.1 latency=3.116369ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.063920268Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:40 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.063944020Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.063965442Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.064169566Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.066252718Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.066271897Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:40.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="974.294µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.068864634Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:40 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.068926073Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.073112839Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.087725386Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.089675970Z" level=debug msg="schedule content cleanup" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.089694416Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:40.102755334Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:40.102877 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:40.102913 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:40.102928 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:40.102976 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"df1c714afb8b85ab827b510e40a545118b18b33c42f4391e170188c8ab38dcae\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:37:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:42Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:42Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.029289206Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.029394664Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.029436697Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.036851237Z" level=debug msg="begin cni setup" podsandboxid=c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:43.053560528Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.18\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ea:bb:31:61:28:30\",\"Sandbox\":\"/var/run/netns/cni-f57db5dd-46b5-d907-fdc5-251856c0355b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-f57db5dd-46b5-d907-fdc5-251856c0355b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth7b23281e\":{\"IPConfigs\":null,\"Mac\":\"c6:e8:f5:75:68:c8\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.055759785Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.055783764Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.057678961Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.057878145Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.057889399Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43Z" level=info handler=mirror ip=127.0.0.1 latency=1.992902ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.061604299Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:43 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.061627829Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.061649513Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.061866236Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.063827477Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.063843261Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:43.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.052855ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.066494301Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:43 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.066533110Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.070485719Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.102484574Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:43.102612 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:43.102645 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:43.102659 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:43.102716 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"c3d5cb011836011128ab071e1cbc547b24529da3b45e4cd926d2f0b8b4c2a577\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.283334989Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.285858581Z" level=debug msg="schedule content cleanup" Mar 13 10:37:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:43.285891760Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:47Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:47Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.030497531Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.030617272Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.030642715Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.038841739Z" level=debug msg="begin cni setup" podsandboxid=14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:52.056669510Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.19\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"1e:e7:86:17:67:f4\",\"Sandbox\":\"/var/run/netns/cni-e1ffd16c-ce2f-3848-22c3-a0c2e94f0075\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-e1ffd16c-ce2f-3848-22c3-a0c2e94f0075\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth02b5b558\":{\"IPConfigs\":null,\"Mac\":\"56:cb:cc:76:ba:68\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.058906324Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.058927502Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.061073661Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.061290914Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.061304372Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52Z" level=info handler=mirror ip=127.0.0.1 latency=3.500461ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.066650158Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:52 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.066676455Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.066698407Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.066911452Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.068980943Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.068999113Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:52.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="912.515µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.071687045Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:52 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.071725611Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.075671316Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.109287716Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:52.109552 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:52.109588 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:52.109601 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:52.109641 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"14f57aa78368fac691f3212e52bca3bc12bf10442ebce0ac3c5fc32a1821234c\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.123663001Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.125673342Z" level=debug msg="schedule content cleanup" Mar 13 10:37:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:52.125704900Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.030508014Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.030607551Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.030627699Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.037466609Z" level=debug msg="begin cni setup" podsandboxid=ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000184bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:37:55.054741014Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.20\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"9a:f8:7f:31:92:0b\",\"Sandbox\":\"/var/run/netns/cni-e4307864-c1ac-4ea9-17d4-95916bb3538c\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-e4307864-c1ac-4ea9-17d4-95916bb3538c\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth8b55654b\":{\"IPConfigs\":null,\"Mac\":\"96:20:63:3a:fc:f1\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.056930252Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.056947540Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.058959460Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.059155387Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.059167966Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55Z" level=info handler=mirror ip=127.0.0.1 latency=2.343771ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.063281585Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:37:55 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.063305206Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.063332303Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.063595888Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.065691159Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.065709679Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:55.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.275907ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.069068716Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:37:55 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.069125495Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.073476811Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.109221810Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:55.109357 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:55.109419 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:55.109440 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:37:55.109493 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"ccb795251d3522393e3df03e5652840645cb9fa03d465fe0e75a488c9dd6a639\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.317124208Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.319606486Z" level=debug msg="schedule content cleanup" Mar 13 10:37:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:55.319635475Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:37:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:56.492Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:37:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:56.492Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:37:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:56.492Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 0 as time since last refresh not above interval Mar 13 10:37:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:56.492Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 1 as time since last refresh not above interval Mar 13 10:37:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:37:56.492Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 2 as time since last refresh not above interval Mar 13 10:37:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:37:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:37:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:37:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:01Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:01Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:02Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:02Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.032647139Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.032772936Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.032798866Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.040467215Z" level=debug msg="begin cni setup" podsandboxid=6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:04.057470343Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.21\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"d6:bd:1e:fa:0e:f9\",\"Sandbox\":\"/var/run/netns/cni-625439ae-06bd-70da-32f9-2a85d2bcede8\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-625439ae-06bd-70da-32f9-2a85d2bcede8\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth507b9ad5\":{\"IPConfigs\":null,\"Mac\":\"3e:b6:cd:07:47:fa\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.059770456Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.059793415Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.062085264Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.062299509Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.062312826Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04Z" level=info handler=mirror ip=127.0.0.1 latency=2.646192ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.066826402Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:04 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.066850691Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.066873793Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.067097224Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.069511982Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.069533566Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.072Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:04.072Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="973.775µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.072184793Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:04 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.072221877Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.076173176Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.107065277Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:04.107192 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:04.107226 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:04.107240 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:04.107280 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"6293ebbc8cda2a27444303903f31a6153ba23f01ca35206476acbc3c00fa5dbe\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.155326499Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.157384365Z" level=debug msg="schedule content cleanup" Mar 13 10:38:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:04.157418383Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.030451462Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.030559439Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.030582977Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.037789103Z" level=debug msg="begin cni setup" podsandboxid=c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:09.054545662Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.22\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"a6:11:6a:0c:f5:a3\",\"Sandbox\":\"/var/run/netns/cni-6ffd2dc4-3658-71a2-4f67-4f29b329885d\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-6ffd2dc4-3658-71a2-4f67-4f29b329885d\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth22b8e669\":{\"IPConfigs\":null,\"Mac\":\"1a:fd:d6:fa:c5:17\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.056735873Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.056757971Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.058693157Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.058914517Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.058928679Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09Z" level=info handler=mirror ip=127.0.0.1 latency=2.278543ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.063092635Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:09 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.063118519Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.063141855Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.063359907Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.065798920Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.065816734Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:09.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="856.085µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.068475523Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:09 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.068520532Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.072479992Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.110247661Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:09.110362 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:09.110414 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:09.110431 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:09.110473 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"c42d0f8765c61dbd959e5d103752379723f7ace711cf8e436da0a37bb4c0ed25\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.316141250Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.318329898Z" level=debug msg="schedule content cleanup" Mar 13 10:38:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:09.318361332Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.029631538Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.029743817Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.029772018Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.037725079Z" level=debug msg="begin cni setup" podsandboxid=11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:15.054817915Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.23\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"1a:4b:2b:a0:79:5a\",\"Sandbox\":\"/var/run/netns/cni-fb15e514-f514-12a9-168f-6a8ae7ac5e8f\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-fb15e514-f514-12a9-168f-6a8ae7ac5e8f\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth940dfe80\":{\"IPConfigs\":null,\"Mac\":\"2e:78:a7:b9:7a:35\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.056958425Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.056978614Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.058902453Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.059112536Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.059124701Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15Z" level=info handler=mirror ip=127.0.0.1 latency=2.902236ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.063844962Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:15 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.063869639Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.063892215Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.064106893Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.066860167Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.066882447Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:15.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="903.184µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.069418491Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:15 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.069487378Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.073663411Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.112274761Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:15.112391 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:15.112438 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:15.112453 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:15.112497 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"11985bf2d0cb321cc1e25ccb7098fa289e798611dad2a6d694cbcfaa5d30636f\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.211732119Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.214294505Z" level=debug msg="schedule content cleanup" Mar 13 10:38:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:15.214325194Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:16 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:16Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:16 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:16Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.029846326Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.029952382Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.029981663Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.037856922Z" level=debug msg="begin cni setup" podsandboxid=dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:22.054771105Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.24\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"f6:12:41:ef:2a:a2\",\"Sandbox\":\"/var/run/netns/cni-d882b68d-46c5-480b-2dc8-67de2875886e\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-d882b68d-46c5-480b-2dc8-67de2875886e\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth7e4bafd0\":{\"IPConfigs\":null,\"Mac\":\"8e:9c:f1:5c:0c:af\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.056937922Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.056960804Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.058942428Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.059182210Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.059196705Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22Z" level=info handler=mirror ip=127.0.0.1 latency=4.03298ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.065165643Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:22 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.065187312Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.065209761Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.065462161Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.067593255Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.067614709Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:22.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.032294ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.070256283Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:22 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.070314504Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.074735552Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.086812937Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.088739438Z" level=debug msg="schedule content cleanup" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.088761605Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:22.108154001Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:22.108268 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:22.108302 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:22.108317 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:22 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:22.108358 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"dc2bc66549d442a27d1b5f81b487c17b07f37c25ae4afcc01bc1d69e4f1af8c2\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:24Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:24Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAHT7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAHT7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000060493} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAAHT7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000076042} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.823Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAACRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.823Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAACRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.000068163} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.823Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAAACRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "time": 0.00007999} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.823Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw=="} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.824Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000050333} Mar 13 10:38:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:24.824Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.00006366} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.029481145Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.029586326Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.029609747Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.036742347Z" level=debug msg="begin cni setup" podsandboxid=afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000184bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:26.053288991Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.25\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"8a:38:55:04:7e:3f\",\"Sandbox\":\"/var/run/netns/cni-928223b7-2bf6-157c-e772-8f26fd2dabee\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-928223b7-2bf6-157c-e772-8f26fd2dabee\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth42945c49\":{\"IPConfigs\":null,\"Mac\":\"b2:8f:a2:5a:31:c2\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.055680255Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.055703622Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.057615633Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.057840769Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.057856264Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26Z" level=info handler=mirror ip=127.0.0.1 latency=2.449437ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.062220071Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:26 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.062245228Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.062267259Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.062525904Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.064668681Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.064688982Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:26.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="949.139µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.067458337Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:26 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.067514499Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.071378537Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.107981213Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:26.108095 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:26.108132 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:26.108147 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:26.108186 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"afa5234df27b9d60bc9df4d333ca7aca2f96a031c4aac120974954bf4ac7a03c\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.261546738Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.263901307Z" level=debug msg="schedule content cleanup" Mar 13 10:38:26 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:26.263941417Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.030553163Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.030658046Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.030680327Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.038711387Z" level=debug msg="begin cni setup" podsandboxid=537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:36.055668932Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.26\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ee:5b:ca:eb:c4:3e\",\"Sandbox\":\"/var/run/netns/cni-ef85118a-4395-a0f1-f519-7ee86eb497c0\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-ef85118a-4395-a0f1-f519-7ee86eb497c0\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethdd57a67a\":{\"IPConfigs\":null,\"Mac\":\"02:42:64:a9:77:89\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.057879612Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.057910198Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.059937434Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.060181056Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.060200697Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36Z" level=info handler=mirror ip=127.0.0.1 latency=2.900538ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.064996215Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:36 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.065018620Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.065043055Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.065258529Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.067431075Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.067452148Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:36.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.06209ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.070124710Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:36 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.070165087Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.074242105Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.084622366Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.086608943Z" level=debug msg="schedule content cleanup" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.086634468Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:36.106447953Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:36.106568 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:36.106603 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:36.106617 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:36.106658 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"537068cef2c34ea3e8cd9b7a14dc29c2db55c84b96e17e24da5f51e0be06d0ce\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:38:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:40Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:40Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.030029785Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.030146745Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.030174795Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.038055397Z" level=debug msg="begin cni setup" podsandboxid=41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:41.054914191Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.27\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"e6:26:a9:2d:9a:c8\",\"Sandbox\":\"/var/run/netns/cni-c41a9797-3973-4c77-6bae-80d462f23592\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-c41a9797-3973-4c77-6bae-80d462f23592\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethdd8e628a\":{\"IPConfigs\":null,\"Mac\":\"b2:34:05:b6:79:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.057167706Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.057190580Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.059191241Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.059457643Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.059472907Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41Z" level=info handler=mirror ip=127.0.0.1 latency=2.237049ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.063610085Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:41 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.063630931Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.063647010Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.063842910Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.066030171Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.066046915Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:41.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="955.112µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.068761200Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:41 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.068827838Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.072881034Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.107445625Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:41.107744 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:41.107779 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:41.107795 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:41.107839 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"41c0a98beda7d62efe4d7426bc4eef04bcb2a5690fb466b108e8d1d1f2f661da\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.243221163Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.245468726Z" level=debug msg="schedule content cleanup" Mar 13 10:38:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:41.245501526Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:44 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:44Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:44 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:44Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.029464627Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.029571219Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.029593057Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.036971462Z" level=debug msg="begin cni setup" podsandboxid=7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000184bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:51.054796179Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.28\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"2e:78:7f:08:f5:bb\",\"Sandbox\":\"/var/run/netns/cni-24fc873a-23e7-b254-6f59-aba72d8d0a29\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-24fc873a-23e7-b254-6f59-aba72d8d0a29\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth089b2cc4\":{\"IPConfigs\":null,\"Mac\":\"0e:0a:51:05:6f:d8\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.057164928Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.057189825Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.059136276Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.059359652Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.059371947Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51Z" level=info handler=mirror ip=127.0.0.1 latency=3.32116ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.064767898Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:51 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.064794062Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.064816011Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.065035164Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.067356145Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.067378366Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:51.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="965.412µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.069986696Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:51 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.070076592Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.074072721Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.114677260Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:51.114804 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:51.114840 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:51.114856 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:51.114900 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"7d5482aad27406f38cb731a46fab5e42a2e556075d65e1e2fa95447e0e60838b\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.312543026Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.314944273Z" level=debug msg="schedule content cleanup" Mar 13 10:38:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:51.314976180Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.029377111Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.029530646Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.029557321Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.036350047Z" level=debug msg="begin cni setup" podsandboxid=2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:38:54.053034290Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.29\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ae:3a:5b:0a:c3:b2\",\"Sandbox\":\"/var/run/netns/cni-70e721a7-c78f-cb84-5850-37092b27daed\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-70e721a7-c78f-cb84-5850-37092b27daed\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethaf9422d2\":{\"IPConfigs\":null,\"Mac\":\"ba:e6:34:2c:f1:7b\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.055282892Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.055315521Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.057226154Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.057463415Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.057475601Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54Z" level=info handler=mirror ip=127.0.0.1 latency=2.404649ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.061750100Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:38:54 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.061767632Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.061791905Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.062003058Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.065226762Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.065248396Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:38:54.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.066298ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.067982288Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:38:54 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.068037790Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.072226819Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.108251098Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:54.108494 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:54.108528 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:54.108544 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:38:54.108582 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"2ee90dea2a9843f5d13b3169819c3bafdb3e6ed9256519f23d05ec16fbc8653b\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.261555441Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.263643378Z" level=debug msg="schedule content cleanup" Mar 13 10:38:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:54.263673496Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:38:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:55Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:55Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:38:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:38:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:38:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.031718162Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.031821059Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.031848796Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.039362041Z" level=debug msg="begin cni setup" podsandboxid=038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:03.056849685Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.30\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ee:16:45:9f:78:2b\",\"Sandbox\":\"/var/run/netns/cni-c0dd862a-c7d1-b765-ff82-824ad27df9c3\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-c0dd862a-c7d1-b765-ff82-824ad27df9c3\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth08d6ca51\":{\"IPConfigs\":null,\"Mac\":\"ea:4b:85:48:38:d6\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.059120853Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.059142760Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.061131238Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.061350920Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.061365651Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03Z" level=info handler=mirror ip=127.0.0.1 latency=3.362287ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.066681786Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:03 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.066705675Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.066729947Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.066949187Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.068969031Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.068986285Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:03.071Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.090951ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.072075344Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:03 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.072112204Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.076421741Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.103778080Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.105669488Z" level=debug msg="schedule content cleanup" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.105751516Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:03.109411686Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:03.109528 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:03.109562 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:03.109576 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:03 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:03.109627 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"038d2fb9f489bd8a22a88aea35faa6ed8be58daede7f171aeef8b7ee1a086f00\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:39:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:05Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:05Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.029737269Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.029850409Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.029872560Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.037157932Z" level=debug msg="begin cni setup" podsandboxid=3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:06.053602298Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.31\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"1e:b8:16:7b:0e:8c\",\"Sandbox\":\"/var/run/netns/cni-171a0889-8586-0d08-53ed-4adc2bfd8035\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-171a0889-8586-0d08-53ed-4adc2bfd8035\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth91303244\":{\"IPConfigs\":null,\"Mac\":\"9a:ed:69:a9:ef:dc\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.055795042Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.055816089Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.057700721Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.057902975Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.057914119Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06Z" level=info handler=mirror ip=127.0.0.1 latency=2.277144ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.062006874Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:06 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.062030408Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.062053099Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.062269587Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.064387297Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.064462395Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:06.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.051543ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.067158801Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:06 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.067228115Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.071094471Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.112678864Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:06.112877 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:06.112914 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:06.112930 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:06.112972 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"3f0523a26ef0e2603c5f6a2a1b09c1143ac7d387efec9e5527b393dfd8441c6a\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.298903708Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.301099316Z" level=debug msg="schedule content cleanup" Mar 13 10:39:06 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:06.301119821Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:07Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:07 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:07Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.029922585Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.030037420Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.030060420Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.037715838Z" level=debug msg="begin cni setup" podsandboxid=f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:14.054686790Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.32\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"36:80:16:f3:af:fe\",\"Sandbox\":\"/var/run/netns/cni-58f91110-cf4e-ac11-b44d-f7a229fa3ee6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-58f91110-cf4e-ac11-b44d-f7a229fa3ee6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vetha04001e7\":{\"IPConfigs\":null,\"Mac\":\"1a:98:5c:b6:4f:3b\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.056987322Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.057010776Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.058947883Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.059172185Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.059185410Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14Z" level=info handler=mirror ip=127.0.0.1 latency=2.601365ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.063625822Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:14 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.063651117Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.063679795Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.063898651Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.066304639Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.066321605Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:14.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="981.769µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.068885244Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:14 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.068942952Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.072959952Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.102993787Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:14.103136 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:14.103180 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:14.103199 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:14.103263 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"f82c370e569b93f15f673a78ecc413e3088f15d5084f98541b28c2c957ed1b9f\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.159215669Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.161440923Z" level=debug msg="schedule content cleanup" Mar 13 10:39:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:14.161474468Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.030138008Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.030256588Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.030281468Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.037519891Z" level=debug msg="begin cni setup" podsandboxid=5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:21.054209996Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.33\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"7e:a7:0a:0e:d5:7f\",\"Sandbox\":\"/var/run/netns/cni-1252e904-7bcf-5802-5d59-148dac7e49ca\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-1252e904-7bcf-5802-5d59-148dac7e49ca\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth91d44375\":{\"IPConfigs\":null,\"Mac\":\"76:f8:a5:e9:9f:3e\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.056647946Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.056669666Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.058671331Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.058888361Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.058901482Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21Z" level=info handler=mirror ip=127.0.0.1 latency=2.517307ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.063323929Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:21 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.063343778Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.063361088Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.063550702Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.065632413Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.065648876Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:21.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="831.473µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.068091383Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:21 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.068120806Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.072303903Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.107415723Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:21.107537 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:21.107570 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:21.107586 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:21.107629 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"5a8c9cb9cb4f1ec7ff7c4421e8da7afd7e1d1b347ea5bf80135448a268227813\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.280842310Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.283216472Z" level=debug msg="schedule content cleanup" Mar 13 10:39:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:21.283252619Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw=="} Mar 13 10:39:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000083222} Mar 13 10:39:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.000100615} Mar 13 10:39:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:25Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:25Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.030427724Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.030529515Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.030550678Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.037900217Z" level=debug msg="begin cni setup" podsandboxid=1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:29.055541894Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.34\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"16:8c:63:88:e3:09\",\"Sandbox\":\"/var/run/netns/cni-bfc6c81d-bb2d-e5e3-9fac-ac823ed82f00\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-bfc6c81d-bb2d-e5e3-9fac-ac823ed82f00\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth59dbe208\":{\"IPConfigs\":null,\"Mac\":\"5e:66:be:39:ed:65\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.057844409Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.057865687Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.059891737Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.060113289Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.060125932Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29Z" level=info handler=mirror ip=127.0.0.1 latency=2.436015ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.064490258Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:29 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.064511246Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.064530804Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.064717655Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.067177750Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.067207381Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:29.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="794.055µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.070421469Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:29 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.070476258Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.075073689Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.105727758Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:29.105848 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:29.105889 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:29.105910 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:29.105979 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"1d8ee17aacf38c07d1f3cc09b572dae96bb9abd9a8ef5f99c04d6284db4184b8\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.140370805Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.142389974Z" level=debug msg="schedule content cleanup" Mar 13 10:39:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:29.142441829Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.030523105Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.030654871Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.030678861Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.038065617Z" level=debug msg="begin cni setup" podsandboxid=e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:34.055026626Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.35\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"0e:be:e3:92:f6:b9\",\"Sandbox\":\"/var/run/netns/cni-ba739737-274a-136e-6797-1c6aca36f977\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-ba739737-274a-136e-6797-1c6aca36f977\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth8c7772d8\":{\"IPConfigs\":null,\"Mac\":\"9a:68:8b:3c:37:49\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.057358249Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.057386557Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.059434237Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.059647071Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.059660653Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34Z" level=info handler=mirror ip=127.0.0.1 latency=2.067195ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.063561731Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:34 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.063585733Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.063613419Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.063835413Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.066045630Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.066063393Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:34.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="815.807µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.068559202Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:34 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.068618118Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.072664397Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.105245695Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:34.105369 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:34.105427 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:34.105444 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:34.105487 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"e9554f50f98511065a17fca759a3a23515237ca915b187ae199502234469008d\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.297675743Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.299950479Z" level=debug msg="schedule content cleanup" Mar 13 10:39:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:34.299984133Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.029344380Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.029465895Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.029487508Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.036460350Z" level=debug msg="begin cni setup" podsandboxid=43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:41.053730962Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.36\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"02:85:a4:4d:d2:0c\",\"Sandbox\":\"/var/run/netns/cni-d22c0541-9508-8950-b096-c04a97059bca\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-d22c0541-9508-8950-b096-c04a97059bca\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth837834f2\":{\"IPConfigs\":null,\"Mac\":\"2e:84:a2:28:d9:27\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.055894457Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.055920732Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.057921239Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.058150541Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.058181904Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41Z" level=info handler=mirror ip=127.0.0.1 latency=2.998929ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.063037874Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:41 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.063060293Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.063079245Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.063267972Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.065407082Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.065428006Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:41.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="837.072µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.068076557Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:41 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.068109403Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.072243132Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.103291837Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:41.103429 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:41.103462 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:41.103477 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:41.103522 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"43333adafa1ad388d18721b7957ece4cfce82a24b2e59f26e4d7f29be5ca079f\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.174592821Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.177061453Z" level=debug msg="schedule content cleanup" Mar 13 10:39:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:41.177098093Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.029466554Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.029586871Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.029610226Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.036964418Z" level=debug msg="begin cni setup" podsandboxid=bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:47.053731424Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.37\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"5e:00:c1:ae:f1:fe\",\"Sandbox\":\"/var/run/netns/cni-1a1e2649-1eb0-e507-abbc-afad070c141e\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-1a1e2649-1eb0-e507-abbc-afad070c141e\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth67e16a86\":{\"IPConfigs\":null,\"Mac\":\"c6:ae:b0:63:06:dc\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.056083154Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.056108422Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.058150557Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.058361325Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.058374710Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47Z" level=info handler=mirror ip=127.0.0.1 latency=3.564375ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.063886062Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:47 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.063912697Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.063943392Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.064168799Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.066472377Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.066492701Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:47.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.083421ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.069268493Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:47 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.069324811Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.073478713Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.114292861Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:47.114446 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:47.114480 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:47.114496 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:47.114535 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"bd1158506b48bc2de25c5188166f7911935762eaeb4552f92e115f035124a6d0\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.316712519Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.318858074Z" level=debug msg="schedule content cleanup" Mar 13 10:39:47 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:47.318889275Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:48 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:48Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:48 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:48Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:50 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:50Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:50 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:50Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.029624586Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.029739054Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.029759049Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.037359092Z" level=debug msg="begin cni setup" podsandboxid=0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:39:56.054831346Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.38\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"12:b2:5a:73:ef:8e\",\"Sandbox\":\"/var/run/netns/cni-f13aefbb-ff3c-0dc9-25e3-452c442436ab\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-f13aefbb-ff3c-0dc9-25e3-452c442436ab\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth7128e73a\":{\"IPConfigs\":null,\"Mac\":\"aa:4a:ef:8a:d3:24\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.057119288Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.057141106Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.059107189Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.059350748Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.059363563Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56Z" level=info handler=mirror ip=127.0.0.1 latency=3.283416ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.064549336Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:39:56 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.064570677Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.064588670Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.064812685Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.066935583Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.066954641Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.014464ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.069629953Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:39:56 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.069706389Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.073823148Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.103171126Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:56.103297 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:56.103330 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:56.103348 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:39:56.103395 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"0bfea0a3de81c5d573c6ac8aa8cf4f5796edbb2d0e797ca67b435ceb8f294ee6\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.158460653Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.160943558Z" level=debug msg="schedule content cleanup" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:56.160974920Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.491Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.491Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.491Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 0 as time since last refresh not above interval Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.491Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 1 as time since last refresh not above interval Mar 13 10:39:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:39:56.491Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 2 as time since last refresh not above interval Mar 13 10:39:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:39:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:39:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:39:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.034557332Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.034668521Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.034692971Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.041751082Z" level=debug msg="begin cni setup" podsandboxid=79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:01.058486409Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.39\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"92:15:d4:59:c3:1e\",\"Sandbox\":\"/var/run/netns/cni-dac83eb2-2f33-e30f-64f6-d679a4645e36\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-dac83eb2-2f33-e30f-64f6-d679a4645e36\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethc63da165\":{\"IPConfigs\":null,\"Mac\":\"a6:95:d2:9d:d3:24\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.060860548Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.060883102Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.062930353Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.063140468Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.063152450Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01Z" level=info handler=mirror ip=127.0.0.1 latency=3.098777ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.068052616Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:01 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.068071182Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.068094288Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.068291339Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.070491289Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.070511531Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.072Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.072Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.072Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:01.072Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.000169ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.073142704Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:01 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.073200309Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.077659371Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.111919883Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:01.112039 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:01.112071 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:01.112091 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:01.112134 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"79d48b9142d928119c3b541aedef164bb2e41ea5195b518213ce3323f52c2a61\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.318279080Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.320135047Z" level=debug msg="schedule content cleanup" Mar 13 10:40:01 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:01.320155243Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:08 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:08Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:08 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:08Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.029547745Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.029662617Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.029707213Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.037469652Z" level=debug msg="begin cni setup" podsandboxid=c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc00011c650), "name":"cbr0", "type":"bridge"} Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:10.055491919Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.40\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"6e:e8:86:38:0f:f7\",\"Sandbox\":\"/var/run/netns/cni-8b08bc2a-1729-1af1-6b43-89abc0dcfdff\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-8b08bc2a-1729-1af1-6b43-89abc0dcfdff\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth1fb7764b\":{\"IPConfigs\":null,\"Mac\":\"2a:5b:4b:11:05:d8\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.057616161Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.057638920Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.059467712Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.059679497Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.059692440Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10Z" level=info handler=mirror ip=127.0.0.1 latency=3.117942ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.064602256Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:10 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.064626492Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.064645231Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.064840962Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.066919179Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.066936002Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:10.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.086836ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.069632341Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:10 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.069670053Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.073595661Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.111873323Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:10.111984 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:10.112015 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:10.112030 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:10.112076 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"c13be0acf3d075ce697fb31ee9451322e2f9fe6d865f763c1233fad5916fd6f5\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.158117290Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.160092193Z" level=debug msg="schedule content cleanup" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10.160112033Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:10 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:10Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.029832111Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.029947994Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.029997142Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.037831168Z" level=debug msg="begin cni setup" podsandboxid=0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:14.054308406Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.41\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"f2:e6:d7:37:7a:73\",\"Sandbox\":\"/var/run/netns/cni-68a5642f-d00d-4cb3-35fe-d5d6e41f15aa\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-68a5642f-d00d-4cb3-35fe-d5d6e41f15aa\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth40674076\":{\"IPConfigs\":null,\"Mac\":\"ce:21:84:bc:ac:b1\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.056580452Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.056601231Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.058607893Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.058811119Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.058824823Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14Z" level=info handler=mirror ip=127.0.0.1 latency=2.406808ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.063080558Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:14 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.063105150Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.063127217Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.063348450Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.065383535Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.065414988Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:14.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="920.384µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.068046260Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:14 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.068111946Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.072325716Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.088041531Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.090023765Z" level=debug msg="schedule content cleanup" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.090044197Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:14.102942068Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:14.103054 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:14.103086 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:14.103101 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:14 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:14.103143 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"0cb2acff152ab47f6194e74bb00cc86c8d34877a0474f2eb534f1e59e58e3fdd\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:40:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.029433645Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.029529570Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.029550765Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.036616016Z" level=debug msg="begin cni setup" podsandboxid=5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:21.053340391Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.42\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"16:d7:15:e4:fd:62\",\"Sandbox\":\"/var/run/netns/cni-a42e7757-3bb5-1f73-ee24-076715d29f6b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-a42e7757-3bb5-1f73-ee24-076715d29f6b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth8e674b9f\":{\"IPConfigs\":null,\"Mac\":\"92:8a:b8:5e:cf:ac\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.055591494Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.055616693Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.057605211Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.057818876Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.057831361Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21Z" level=info handler=mirror ip=127.0.0.1 latency=3.122ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.062885777Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:21 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.062916452Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.062940038Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.063152886Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.065385940Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.065421086Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:21.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.350692ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.068653394Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:21 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.068733246Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.072897926Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.113035109Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:21.113236 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:21.113271 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:21.113288 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:21.113332 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"5bdd27557be14b46e2a5e0552dde83fddde259326d292e82f707dcaa022f3f88\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.212423908Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.214699766Z" level=debug msg="schedule content cleanup" Mar 13 10:40:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:21.214731261Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:24.823Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:62 error reading message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "error": "stream reset (local): code: 0x0: transport error: stream reset, error code: 0"} Mar 13 10:40:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:24Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:24Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.029194151Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.029313514Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.029337614Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.036229111Z" level=debug msg="begin cni setup" podsandboxid=2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:25.052537749Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.43\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"86:27:f1:e3:91:21\",\"Sandbox\":\"/var/run/netns/cni-0df2c42d-597d-206a-41a0-54f0b3de4653\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-0df2c42d-597d-206a-41a0-54f0b3de4653\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth14bd73e0\":{\"IPConfigs\":null,\"Mac\":\"de:5c:df:37:66:a5\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.054799496Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.054831534Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.056760148Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.056994878Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.057008809Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.059Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25Z" level=info handler=mirror ip=127.0.0.1 latency=1.668969ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.060546644Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:25 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.060572094Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.060592503Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.060807725Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.062887205Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.062907183Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.064Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:25.065Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.171789ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.065642423Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:25 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.065706035Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.069654771Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.100207054Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:25.100333 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:25.100368 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:25.100383 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:25.100444 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"2c37449896039eab98ded21e571646aa204b9e05702849f1aebcfb5a154b61ba\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.143452336Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.145546133Z" level=debug msg="schedule content cleanup" Mar 13 10:40:25 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:25.145566078Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.030527342Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.030636059Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.030657448Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.038308942Z" level=debug msg="begin cni setup" podsandboxid=b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:35.055757507Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.44\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ce:34:0b:c2:16:d9\",\"Sandbox\":\"/var/run/netns/cni-6ef8a427-db0a-b9fc-82c0-c5fd5fa8a77b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-6ef8a427-db0a-b9fc-82c0-c5fd5fa8a77b\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth2c67b851\":{\"IPConfigs\":null,\"Mac\":\"36:7e:f9:08:86:1f\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.058008328Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.058030469Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.060020853Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.060230044Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.060243352Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35Z" level=info handler=mirror ip=127.0.0.1 latency=3.089912ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.065213029Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:35 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.065237292Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.065268891Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.065495801Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.068050891Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.068071944Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:35.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.047247ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.070724835Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:35 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.070806567Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.075416162Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.115093952Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:35.115216 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:35.115253 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:35.115268 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:35.115311 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"b629af560c17536370d260a404dd5320e3fc32c9d6f77f198fe69cdc47ef3293\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.212423910Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.214740448Z" level=debug msg="schedule content cleanup" Mar 13 10:40:35 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:35.214768842Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:36Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:36 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:36Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.030129487Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.030235187Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.030260022Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.037212676Z" level=debug msg="begin cni setup" podsandboxid=9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:38.054302248Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.45\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"7e:ba:10:dd:eb:89\",\"Sandbox\":\"/var/run/netns/cni-e95d4b04-3289-ad1c-66df-b3cf245695af\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-e95d4b04-3289-ad1c-66df-b3cf245695af\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth4da00756\":{\"IPConfigs\":null,\"Mac\":\"1e:42:84:7c:6c:a9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.056573405Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.056594786Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.058457357Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.058676041Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.058688207Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38Z" level=info handler=mirror ip=127.0.0.1 latency=2.063893ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.062542188Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:38 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.062564926Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.062586829Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.062804158Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.065698068Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.065720303Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:38.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="818.021µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.068106560Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:38 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.068173053Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.072867817Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.105593775Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:38.105703 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:38.105735 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:38.105748 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:38.105788 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"9e5b211caa015887ef9a2ec1fdde73896aeb0225ee871835720e5cc8445fe8c7\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.160852068Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.163007194Z" level=debug msg="schedule content cleanup" Mar 13 10:40:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:38.163037363Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:43Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:43 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:43Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.029264926Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.029371544Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.029418442Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.037019182Z" level=debug msg="begin cni setup" podsandboxid=8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:46.053789591Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.46\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"e2:36:f0:ba:58:57\",\"Sandbox\":\"/var/run/netns/cni-2e74bd1f-c034-6580-a69a-3df003cf9da6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-2e74bd1f-c034-6580-a69a-3df003cf9da6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth21c43b11\":{\"IPConfigs\":null,\"Mac\":\"b6:48:da:8a:55:e8\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.055880060Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.055901565Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.057869659Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.058112472Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.058125665Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46Z" level=info handler=mirror ip=127.0.0.1 latency=3.437781ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.063426210Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:46 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.063453177Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.063480580Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.063702826Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.066116557Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.066139259Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:46.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="975.064µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.068696922Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:46 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.068747090Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.072812125Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.108853074Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:46.109086 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:46.109127 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:46.109149 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:46.109201 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"8ab9f127112c143e6a9013d10457610e81b18618428056b552c25f420feb55ed\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.267301808Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.269786495Z" level=debug msg="schedule content cleanup" Mar 13 10:40:46 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:46.269817358Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.029927850Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.030051735Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.030078902Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.037531675Z" level=debug msg="begin cni setup" podsandboxid=66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:51.054696758Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.47\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"aa:f8:25:d1:96:fa\",\"Sandbox\":\"/var/run/netns/cni-722f8469-cbee-d0b6-62c1-12aa5f774f30\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-722f8469-cbee-d0b6-62c1-12aa5f774f30\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vetha031e29d\":{\"IPConfigs\":null,\"Mac\":\"92:f2:9c:38:f7:46\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.057071941Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.057095572Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.059099926Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.059315886Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.059329107Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51Z" level=info handler=mirror ip=127.0.0.1 latency=3.591858ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.064822645Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:51 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.064848375Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.064870345Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.065085103Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.067117781Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.067134825Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:51.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.367827ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.070051466Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:51 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.070086281Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.074086326Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.115953554Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:51.116066 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:51.116098 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:51.116115 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:51.116154 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"66e2f0f39a53b5912eb3286765b7ce00f4dc241dedb90da23f73bf4b02c81fa7\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.181215194Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.183313827Z" level=debug msg="schedule content cleanup" Mar 13 10:40:51 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:51.183335601Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:54Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:54Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:55Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:55Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.029375873Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.029492532Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.029517806Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.037137203Z" level=debug msg="begin cni setup" podsandboxid=3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:40:57.054204299Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.48\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"5a:32:1d:a1:40:bf\",\"Sandbox\":\"/var/run/netns/cni-4a7969d9-f522-92eb-c3cb-68cb9f612399\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-4a7969d9-f522-92eb-c3cb-68cb9f612399\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth4a3ab5f6\":{\"IPConfigs\":null,\"Mac\":\"ba:1d:27:77:b3:2d\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.056516644Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.056539120Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.058567838Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.058773509Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.058787188Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57Z" level=info handler=mirror ip=127.0.0.1 latency=3.058015ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.063687231Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:40:57 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.063713565Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.063738423Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.063948447Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.066091781Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.066109269Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:40:57.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.135048ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.068921353Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:40:57 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.068977673Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.073153189Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.077787265Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.079728932Z" level=debug msg="schedule content cleanup" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.079747907Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:57.106565522Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:57.106685 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:57.106716 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:57.106730 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:40:57 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:40:57.106774 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"3ea008a4ad79c625f4e6bea7e35d7d9b2f6f794461c11ca5562c0d8b7c142671\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:40:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:40:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:40:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:40:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:04Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:04 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:04Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.032871542Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.032967721Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.032991032Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.040932884Z" level=debug msg="begin cni setup" podsandboxid=403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:05.059930503Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.49\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"82:6a:3a:a9:e7:cc\",\"Sandbox\":\"/var/run/netns/cni-5ac7396d-5401-73a1-9ad4-155c15dca400\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-5ac7396d-5401-73a1-9ad4-155c15dca400\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth6192b675\":{\"IPConfigs\":null,\"Mac\":\"be:ea:9e:e5:5e:37\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.062076653Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.062098172Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.063999836Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.064228136Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.064240580Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05Z" level=info handler=mirror ip=127.0.0.1 latency=3.01711ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.069133618Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:05 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.069157440Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.069178869Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.069383586Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.071512726Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.071532176Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.073Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.073Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.073Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:05.073Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="907.85µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.074167321Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:05 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.074234020Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.078270787Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.109087131Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:05.109233 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:05.109268 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:05.109287 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:05.109341 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"403e6d943cbd37ed2c436930b2f9212f033b4420f720228c7d9c8c79d48aef9e\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.183509022Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.185779097Z" level=debug msg="schedule content cleanup" Mar 13 10:41:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:05.185810136Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.029906434Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.030027377Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.030050525Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.037362893Z" level=debug msg="begin cni setup" podsandboxid=f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:12.054534041Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.50\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ae:7b:ff:29:c5:10\",\"Sandbox\":\"/var/run/netns/cni-c9059edf-022c-ebf1-6214-7e5fc7b2eaed\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-c9059edf-022c-ebf1-6214-7e5fc7b2eaed\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vetheb088218\":{\"IPConfigs\":null,\"Mac\":\"8e:a3:64:94:5f:79\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.056773111Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.056793952Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.058667083Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.058880938Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.058893297Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12Z" level=info handler=mirror ip=127.0.0.1 latency=3.553857ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.064306311Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:12 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.064327836Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.064347486Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.064560233Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.066649744Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.066669440Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:12.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.051424ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.069423379Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:12 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.069486854Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.073745544Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.111115281Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:12.111246 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:12.111280 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:12.111295 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:12.111337 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"f710b459ae701caa4629eabe3fcaa3c43cae864ef3d87bceba885cde033c4449\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.307530160Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.310030186Z" level=debug msg="schedule content cleanup" Mar 13 10:41:12 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:12.310062368Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:16 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:16Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:16 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:16Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:17 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:17Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:17 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:17Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.030474670Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.030592290Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.030615806Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.037844307Z" level=debug msg="begin cni setup" podsandboxid=727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:20.055082333Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.51\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"86:83:3e:5d:46:fe\",\"Sandbox\":\"/var/run/netns/cni-4452a89f-cfb0-64ae-a09b-24ffd0ca0cf2\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-4452a89f-cfb0-64ae-a09b-24ffd0ca0cf2\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth0fde1c63\":{\"IPConfigs\":null,\"Mac\":\"3a:2c:57:d1:d5:4f\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.057475898Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.057498503Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.059494006Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.059722654Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.059735870Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20Z" level=info handler=mirror ip=127.0.0.1 latency=2.879434ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.064516138Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:20 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.064540651Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.064562332Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.064813728Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.066884725Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.066902009Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:20.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.146769ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.069614347Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:20 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.069662818Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.074087115Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.112323869Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:20.112471 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:20.112510 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:20.112534 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:20.112591 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"727f1f1a689f857452616654b3f031a1b50978d9440d7e0ae7b6c7ac0c088398\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.166573205Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.168636897Z" level=debug msg="schedule content cleanup" Mar 13 10:41:20 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:20.168655345Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:106 handling message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw=="} Mar 13 10:41:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:123 handled message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.00007643} Mar 13 10:41:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:24.822Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:149 responded to message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "type": 4, "key": "EiAdPANQ1JDV7yD6cfw/YRKdgsEdR4aactk1KHwrdKSBdw==", "time": 0.00009212} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.030510659Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.030623494Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.030643678Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.038368034Z" level=debug msg="begin cni setup" podsandboxid=0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:27.055120563Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.52\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"3a:74:aa:c4:87:11\",\"Sandbox\":\"/var/run/netns/cni-053852f2-a6b7-0a89-da85-47db4e51ff39\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-053852f2-a6b7-0a89-da85-47db4e51ff39\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vetha9d43b8b\":{\"IPConfigs\":null,\"Mac\":\"4e:6c:85:97:7f:6b\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.057465529Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.057495068Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.059458228Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.059678646Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.059691023Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27Z" level=info handler=mirror ip=127.0.0.1 latency=2.902979ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.064484925Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:27 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.064510673Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.064532622Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.064757017Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.066935016Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.066955369Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:27.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="760.251µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.069323274Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:27 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.069361616Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.073864088Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.110472856Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:27.110589 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:27.110620 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:27.110637 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:27.110679 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"0dd280dbaeff90769e931d6637f2ebf716de5761cd82b4dc238cca69e956b9ac\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.289864007Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.291893811Z" level=debug msg="schedule content cleanup" Mar 13 10:41:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:27.291919478Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:28Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:28 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:28Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:31 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:31Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:31 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:31Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.029585069Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.029703982Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.029727753Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.037334412Z" level=debug msg="begin cni setup" podsandboxid=19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:34.054115549Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.53\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ba:d7:e2:e5:62:93\",\"Sandbox\":\"/var/run/netns/cni-2d60a9ee-e48a-1424-ef68-2a3b892073b8\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-2d60a9ee-e48a-1424-ef68-2a3b892073b8\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth14fad5fd\":{\"IPConfigs\":null,\"Mac\":\"9a:a3:db:02:c6:ff\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.056468892Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.056490327Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.058540175Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.058753023Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.058766519Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34Z" level=info handler=mirror ip=127.0.0.1 latency=3.353392ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.063985196Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:34 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.064007491Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.064027825Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.064218847Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.066378226Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.066412671Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:34.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="825.281µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.068864467Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:34 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.068904112Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.073058569Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.109451658Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:34.109573 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:34.109604 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:34.109619 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:34.109656 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"19eb5da4c227907703d8b20deb0fc301bbc9ad3131181bdababda3e72b2a52ae\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.168723645Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.170788076Z" level=debug msg="schedule content cleanup" Mar 13 10:41:34 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:34.170820876Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.030829273Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.030935497Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.030956777Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.038935003Z" level=debug msg="begin cni setup" podsandboxid=a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000118bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:39.055959900Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.54\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"1a:28:cc:12:71:9d\",\"Sandbox\":\"/var/run/netns/cni-0288f7eb-9a56-84ed-a007-4c7c6c4a6aab\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-0288f7eb-9a56-84ed-a007-4c7c6c4a6aab\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth02c667c3\":{\"IPConfigs\":null,\"Mac\":\"26:0b:a7:b4:49:0d\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.058051205Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.058074144Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.059971840Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.060192881Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.060204954Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.063Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=info handler=mirror ip=127.0.0.1 latency=3.332356ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.065345059Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:39 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.065366023Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.065388320Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.065598491Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.067589030Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.067605397Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:39.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.145135ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.070548182Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:39 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.070634678Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.074585638Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.081352425Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.083345517Z" level=debug msg="schedule content cleanup" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.083377053Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39.108317548Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:39.108508 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:39.108546 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:39.108561 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:39.108601 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"a9d4711816bca41e1de0ba5e30668cceae7b9334aabbd4cbf44f3a8434cb5a43\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233775 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"detector-config\" (UniqueName: \"kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-detector-config\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233821 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-certificates\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233837 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"secrets-data\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-secrets-data\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233852 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp-cc-detector\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233867 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp-cc-detector-mlservice\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector-mlservice\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233881 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-trust-bundle\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233895 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-mkd7q\" (UniqueName: \"kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-kube-api-access-mkd7q\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233936 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"dnld-storage\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-dnld-storage\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.233965 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-config\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.234004 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared-data\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-data\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.234018 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"local-storage\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-local-storage\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.234029 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"shared-socket-directory\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-socket-directory\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.234041 6408 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp-download-worker\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-download-worker\") pod \"priv-appliance-download-worker-master-7895b5d47d-srdbc\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:42.285794 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="unmounted volumes=[config detector-config dnld-storage kube-api-access-mkd7q local-storage secrets-data shared-certificates shared-data shared-socket-directory tmp-cc-detector tmp-cc-detector-mlservice tmp-download-worker trust-bundle], unattached volumes=[], failed to process volumes=[]: context canceled" pod="default/priv-appliance-download-worker-master-7895b5d47d-srdbc" podUID="70c28abf-08c3-487c-9cfa-4347186a7e9e" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635854 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-certificates\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635876 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-trust-bundle\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635888 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"config\" (UniqueName: \"kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-config\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635898 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"shared-data\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-data\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635910 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"local-storage\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-local-storage\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635918 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"tmp-download-worker\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-download-worker\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635928 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"detector-config\" (UniqueName: \"kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-detector-config\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635936 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"secrets-data\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-secrets-data\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635945 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"tmp-cc-detector-mlservice\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector-mlservice\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635955 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"dnld-storage\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-dnld-storage\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635966 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"shared-socket-directory\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-socket-directory\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635976 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"kube-api-access-mkd7q\" (UniqueName: \"kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-kube-api-access-mkd7q\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.635984 6408 reconciler_common.go:163] "operationExecutor.UnmountVolume started for volume \"tmp-cc-detector\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector\") pod \"70c28abf-08c3-487c-9cfa-4347186a7e9e\" (UID: \"70c28abf-08c3-487c-9cfa-4347186a7e9e\") " Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636125 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector" (OuterVolumeSpecName: "tmp-cc-detector") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "tmp-cc-detector". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636165 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-config" (OuterVolumeSpecName: "config") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "config". PluginName "kubernetes.io/configmap", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636207 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-data" (OuterVolumeSpecName: "shared-data") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "shared-data". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636246 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-certificates" (OuterVolumeSpecName: "shared-certificates") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "shared-certificates". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636258 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-secrets-data" (OuterVolumeSpecName: "secrets-data") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "secrets-data". PluginName "kubernetes.io/host-path", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636296 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-local-storage" (OuterVolumeSpecName: "local-storage") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "local-storage". PluginName "kubernetes.io/host-path", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636333 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector-mlservice" (OuterVolumeSpecName: "tmp-cc-detector-mlservice") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "tmp-cc-detector-mlservice". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636368 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-dnld-storage" (OuterVolumeSpecName: "dnld-storage") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "dnld-storage". PluginName "kubernetes.io/host-path", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636370 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-socket-directory" (OuterVolumeSpecName: "shared-socket-directory") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "shared-socket-directory". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636416 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-download-worker" (OuterVolumeSpecName: "tmp-download-worker") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "tmp-download-worker". PluginName "kubernetes.io/empty-dir", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636507 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-detector-config" (OuterVolumeSpecName: "detector-config") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "detector-config". PluginName "kubernetes.io/configmap", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.636956 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-trust-bundle" (OuterVolumeSpecName: "trust-bundle") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "trust-bundle". PluginName "kubernetes.io/projected", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.637068 6408 operation_generator.go:781] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-kube-api-access-mkd7q" (OuterVolumeSpecName: "kube-api-access-mkd7q") pod "70c28abf-08c3-487c-9cfa-4347186a7e9e" (UID: "70c28abf-08c3-487c-9cfa-4347186a7e9e"). InnerVolumeSpecName "kube-api-access-mkd7q". PluginName "kubernetes.io/projected", VolumeGIDValue "" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736757 6408 reconciler_common.go:299] "Volume detached for volume \"tmp-cc-detector-mlservice\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector-mlservice\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736775 6408 reconciler_common.go:299] "Volume detached for volume \"dnld-storage\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-dnld-storage\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736780 6408 reconciler_common.go:299] "Volume detached for volume \"shared-socket-directory\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-socket-directory\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736786 6408 reconciler_common.go:299] "Volume detached for volume \"kube-api-access-mkd7q\" (UniqueName: \"kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-kube-api-access-mkd7q\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736791 6408 reconciler_common.go:299] "Volume detached for volume \"tmp-cc-detector\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-cc-detector\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736796 6408 reconciler_common.go:299] "Volume detached for volume \"shared-certificates\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-certificates\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736801 6408 reconciler_common.go:299] "Volume detached for volume \"trust-bundle\" (UniqueName: \"kubernetes.io/projected/70c28abf-08c3-487c-9cfa-4347186a7e9e-trust-bundle\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736805 6408 reconciler_common.go:299] "Volume detached for volume \"config\" (UniqueName: \"kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-config\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736810 6408 reconciler_common.go:299] "Volume detached for volume \"shared-data\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-shared-data\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736815 6408 reconciler_common.go:299] "Volume detached for volume \"local-storage\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-local-storage\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736820 6408 reconciler_common.go:299] "Volume detached for volume \"tmp-download-worker\" (UniqueName: \"kubernetes.io/empty-dir/70c28abf-08c3-487c-9cfa-4347186a7e9e-tmp-download-worker\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736824 6408 reconciler_common.go:299] "Volume detached for volume \"detector-config\" (UniqueName: \"kubernetes.io/configmap/70c28abf-08c3-487c-9cfa-4347186a7e9e-detector-config\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:42.736828 6408 reconciler_common.go:299] "Volume detached for volume \"secrets-data\" (UniqueName: \"kubernetes.io/host-path/70c28abf-08c3-487c-9cfa-4347186a7e9e-secrets-data\") on node \"ip-10-151-21-17.us-west-2.compute.internal\" DevicePath \"\"" Mar 13 10:41:45 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: I0313 10:41:45.028611 6408 kubelet_volumes.go:163] "Cleaned up orphaned pod volumes dir" podUID="70c28abf-08c3-487c-9cfa-4347186a7e9e" path="/var/lib/kubelet/pods/70c28abf-08c3-487c-9cfa-4347186a7e9e/volumes" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.035202641Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.035320627Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.035345792Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.042658387Z" level=debug msg="begin cni setup" podsandboxid=8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000184bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:49.059548099Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.55\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"fa:05:59:c3:db:d1\",\"Sandbox\":\"/var/run/netns/cni-7cb0889b-40f2-3e28-ba58-0f1a515bad13\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-7cb0889b-40f2-3e28-ba58-0f1a515bad13\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth72205080\":{\"IPConfigs\":null,\"Mac\":\"9e:0d:6e:a1:c3:58\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.061802191Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.061824839Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.063789476Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.064003326Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.064015368Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49Z" level=info handler=mirror ip=127.0.0.1 latency=2.823883ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.069161594Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:49 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.069189086Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.069212064Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.069487121Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.071667017Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.071686250Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.073Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.074Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.074Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:49.074Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="961.046µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.074275566Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:49 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.074347216Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.078347899Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.113051147Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:49.113272 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:49.113304 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:49.113320 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:49.113360 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"8af58c94b141d7204437bb195d5a7dcfccf7af53511bec435e53c621b5ca27ec\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.150389739Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.152851427Z" level=debug msg="schedule content cleanup" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49.152885106Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:49 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:49Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:52Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:52 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:52Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.030418311Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.030531256Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.030556278Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.038393075Z" level=debug msg="begin cni setup" podsandboxid=6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:41:54.055184887Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.56\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"be:2b:ea:cd:37:71\",\"Sandbox\":\"/var/run/netns/cni-6a74412f-5b10-5aae-fdd6-aa6dd41d35b4\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-6a74412f-5b10-5aae-fdd6-aa6dd41d35b4\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth516436c3\":{\"IPConfigs\":null,\"Mac\":\"22:94:b4:ec:ba:6f\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.057490397Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.057513217Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.059477566Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.059691756Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.059703826Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54Z" level=info handler=mirror ip=127.0.0.1 latency=1.726107ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.063341128Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:41:54 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.063366145Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.063391155Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.063636197Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.065831998Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.065850305Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:54.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.001693ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.068484816Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:41:54 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.068540808Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.072524092Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.108733976Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:54.108854 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:54.108889 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:54.108905 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:41:54.108945 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"6edd5a35df1ae9115fc1637adff6fb42815155849b2309a8670c4ffdc030016e\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.311133405Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.313642622Z" level=debug msg="schedule content cleanup" Mar 13 10:41:54 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:54.313672982Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:41:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:55Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:55 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:55Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:56.492Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:41:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:56.492Z DEBUG dht/netsize netsize/netsize.go:258 Running garbage collection Mar 13 10:41:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:56.492Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 0 as time since last refresh not above interval Mar 13 10:41:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:56.492Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 1 as time since last refresh not above interval Mar 13 10:41:56 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:41:56.492Z DEBUG dht/RtRefreshManager rtrefresh/rt_refresh_manager.go:304 not running refresh for cpl 2 as time since last refresh not above interval Mar 13 10:41:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:41:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:41:59 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:41:59Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.033738902Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.033858697Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.033883877Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.042466263Z" level=debug msg="begin cni setup" podsandboxid=59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc0000b0bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:02.059527155Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.57\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"06:b0:27:85:9b:5e\",\"Sandbox\":\"/var/run/netns/cni-c3be5f34-7920-344d-ef99-3ab28f4ca174\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-c3be5f34-7920-344d-ef99-3ab28f4ca174\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth34b4ca5d\":{\"IPConfigs\":null,\"Mac\":\"82:c9:f4:51:48:8c\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.061794247Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.061817080Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.063824265Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.064056685Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.064068866Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02Z" level=info handler=mirror ip=127.0.0.1 latency=2.927172ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.069039835Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:02 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.069067471Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.069106870Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.069325749Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.071448563Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.071469485Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.073Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.074Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.074Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:02.074Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.102544ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.074375307Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:02 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.074445259Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.078562883Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.108349251Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:02.108522 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:02.108560 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:02.108579 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:02.108624 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"59074fb66ca8f01fdc9ef0a408b14e0c3ddbaf105fbaaa684d207b35495e205a\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.170816652Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.173009011Z" level=debug msg="schedule content cleanup" Mar 13 10:42:02 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:02.173028568Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.029892271Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.030011591Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.030039436Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.037379634Z" level=debug msg="begin cni setup" podsandboxid=bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:05.054321203Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.58\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"7a:66:4c:39:b1:02\",\"Sandbox\":\"/var/run/netns/cni-d2767a7f-0285-bb20-e182-b5d8df92dba4\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-d2767a7f-0285-bb20-e182-b5d8df92dba4\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethe37e0411\":{\"IPConfigs\":null,\"Mac\":\"22:ed:d0:06:d4:24\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.056522175Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.056542933Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.058497744Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.058703830Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.058715996Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05Z" level=info handler=mirror ip=127.0.0.1 latency=2.189626ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.062697727Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:05 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.062722888Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.062750502Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.062978549Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.065296013Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.065319557Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:05.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="921.595µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.067932850Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:05 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.068007930Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.072022240Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.102370784Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:05.102511 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:05.102545 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:05.102560 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:05.102600 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"bfde2e57e380ce042eb342d4260a35510bb05da727ae7fc0e7407034b620615b\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.118632336Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.120698814Z" level=debug msg="schedule content cleanup" Mar 13 10:42:05 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:05.120716730Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:42:08 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:08Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:08 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:08Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:09 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:09Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.029483993Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.029599135Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.029620949Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.037033999Z" level=debug msg="begin cni setup" podsandboxid=046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000184bc0), "name":"cbr0", "type":"bridge"} Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:15.053874554Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.59\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"ba:1d:9e:e9:e4:ab\",\"Sandbox\":\"/var/run/netns/cni-9ea845d0-eb95-2b99-0320-68fa026b9cc6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-9ea845d0-eb95-2b99-0320-68fa026b9cc6\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth5f2a77e9\":{\"IPConfigs\":null,\"Mac\":\"ca:33:1e:1e:6e:c7\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.056139705Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.056166393Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.058172769Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.058392614Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.058422026Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15Z" level=info handler=mirror ip=127.0.0.1 latency=2.533805ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.062804621Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:15 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.062827638Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.062847332Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.063062839Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.065803589Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.065830555Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:15.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.198345ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.068750269Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:15 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.068811508Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.072941278Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.103283324Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:15.103412 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:15.103446 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:15.103461 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:15.103500 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"046d6095d85fdf963018e63dceae76b0b82156ffa01a2c141a0d0d2d6991621d\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.184579359Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.186657095Z" level=debug msg="schedule content cleanup" Mar 13 10:42:15 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:15.186691896Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.029599221Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.029711760Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.029733006Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.036565325Z" level=debug msg="begin cni setup" podsandboxid=b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:18.053340888Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.60\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"82:4a:37:7b:fb:75\",\"Sandbox\":\"/var/run/netns/cni-59a9de45-9535-be0b-7b09-f342b861efc2\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-59a9de45-9535-be0b-7b09-f342b861efc2\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethd2f60b32\":{\"IPConfigs\":null,\"Mac\":\"c2:29:a3:d8:bc:b7\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.055632787Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.055656322Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.057865337Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.058080081Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.058093532Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18Z" level=info handler=mirror ip=127.0.0.1 latency=2.239198ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.062236517Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:18 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.062257728Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.062284161Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.062504350Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.064583400Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.064602852Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:18.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency="855.279µs" logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.067292641Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:18 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.067366758Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.071372901Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.118245403Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:18.118363 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:18.118417 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:18.118436 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:18.118478 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"b88d27687bd675325b1e5112844c7be9b8903f4ce221626c5c25aefccbe434f2\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.133512037Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.135364959Z" level=debug msg="schedule content cleanup" Mar 13 10:42:18 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:18.135384729Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:42:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:19 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:19Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:21Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:21 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:21Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:24 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:24.823Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht_net.go:62 error reading message {"from": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt", "error": "stream reset (local): code: 0x0: transport error: stream reset, error code: 0"} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.029547902Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.029668259Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.029692884Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.037881564Z" level=debug msg="begin cni setup" podsandboxid=1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:27.055270774Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.61\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"4a:ee:37:b5:85:0a\",\"Sandbox\":\"/var/run/netns/cni-701774f8-8e2c-7b10-06a6-e20e5bc782a2\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-701774f8-8e2c-7b10-06a6-e20e5bc782a2\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth7d1c3ea0\":{\"IPConfigs\":null,\"Mac\":\"b2:cb:dc:0c:14:20\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.057526337Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.057553182Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.059618677Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.059846010Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.059859220Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.062Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27Z" level=info handler=mirror ip=127.0.0.1 latency=3.101792ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.064870150Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:27 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.064896395Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.064916395Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.065133011Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.067441292Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.067463191Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.069Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:27.070Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.134399ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.070179485Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:27 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.070218168Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.074088389Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.105232989Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:27.105469 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:27.105506 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:27.105519 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:27.105562 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"1984808b828e2c6101ff4c714f28fe67060a48663eb07ed0590636e5f3b0c8ad\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.219653911Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.221703850Z" level=debug msg="schedule content cleanup" Mar 13 10:42:27 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:27.221732292Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:42:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:29 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:29Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.029474871Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.029573283Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.029590985Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.036797284Z" level=debug msg="begin cni setup" podsandboxid=c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:30.053677494Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.62\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"de:b3:de:e8:a2:95\",\"Sandbox\":\"/var/run/netns/cni-834d6b95-3ff3-e517-4e80-921cd7dd54db\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-834d6b95-3ff3-e517-4e80-921cd7dd54db\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethaeee1863\":{\"IPConfigs\":null,\"Mac\":\"de:ce:28:b3:75:8c\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.056073593Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.056091871Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.058181855Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.058389855Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.058415019Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.060Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30Z" level=info handler=mirror ip=127.0.0.1 latency=1.740958ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.061915351Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:30 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.061937513Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.061958875Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.062195534Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.064361289Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.064381579Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.066Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:30.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.039706ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.067231490Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:30 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.067292107Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.071483102Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.103148575Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:30.103269 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:30.103301 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:30.103316 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:30.103356 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-monitor-status-zm4mj_default(4f679831-c7be-46a9-bab6-ea408cd29c72)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"c7cb8b62eba57ca80a8881f0cb2156011bf7562e4e7f08b34584d54b3fa930fe\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-monitor-status-zm4mj" podUID="4f679831-c7be-46a9-bab6-ea408cd29c72" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.167510867Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.169882924Z" level=debug msg="schedule content cleanup" Mar 13 10:42:30 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:30.169912887Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:42:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:38Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:38 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:38Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:39 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:39Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.029547508Z" level=info msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.029670862Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\"} hostname:\"priv-appliance-prometheus-node-exporter-qtwlg\" log_directory:\"/var/log/pods/default_priv-appliance-prometheus-node-exporter-qtwlg_48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} port_mappings:{container_port:9100} labels:{key:\"app.kubernetes.io/component\" value:\"metrics\"} labels:{key:\"app.kubernetes.io/instance\" value:\"priv-appliance\"} labels:{key:\"app.kubernetes.io/managed-by\" value:\"Helm\"} labels:{key:\"app.kubernetes.io/name\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/part-of\" value:\"prometheus-node-exporter\"} labels:{key:\"app.kubernetes.io/version\" value:\"1.6.1\"} labels:{key:\"app.securiti.ai/service-name\" value:\"prometheus-node-exporter\"} labels:{key:\"controller-revision-hash\" value:\"84c564dbc9\"} labels:{key:\"helm.sh/chart\" value:\"prometheus-node-exporter-4.23.2\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-prometheus-node-exporter-qtwlg\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\"} labels:{key:\"pod-template-generation\" value:\"1\"} annotations:{key:\"cluster-autoscaler.kubernetes.io/safe-to-evict\" value:\"true\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014017717Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod48d05fd1_9744_41fd_83cb_f32cb8b2ed30.slice\" security_context:{namespace_options:{pid:NODE userns_options:{mode:NODE}} run_as_user:{value:65534} run_as_group:{value:65534} supplemental_groups:65534 seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:10000 cpu_shares:10 memory_limit_in_bytes:188743680 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.029696222Z" level=debug msg="generated id for sandbox name \"priv-appliance-prometheus-node-exporter-qtwlg_default_48d05fd1-9744-41fd-83cb-f32cb8b2ed30_0\"" podsandboxid=e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.037361729Z" level=debug msg="begin cni setup" podsandboxid=e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:40.054486950Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.63\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"8e:d4:79:39:6c:d3\",\"Sandbox\":\"/var/run/netns/cni-4686b5c0-e691-fe33-139d-9f14ee164d7a\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-4686b5c0-e691-fe33-139d-9f14ee164d7a\",\"PciID\":\"\",\"SocketPath\":\"\"},\"vethc5c02676\":{\"IPConfigs\":null,\"Mac\":\"d6:fb:d6:08:d1:e8\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.056774993Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.056796982Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.058781984Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.058990802Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.059004953Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40Z" level=info handler=mirror ip=127.0.0.1 latency=2.750353ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.063627080Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:40 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.063652094Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.063673925Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.063881291Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.066037581Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.066055643Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:40.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.127234ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.068918097Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:40 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.068991165Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.073485725Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.114183056Z" level=error msg="RunPodSandbox for name:\"priv-appliance-prometheus-node-exporter-qtwlg\" uid:\"48d05fd1-9744-41fd-83cb-f32cb8b2ed30\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:40.114312 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:40.114344 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:40.114359 6408 kuberuntime_manager.go:1343] "CreatePodSandbox for pod failed" err="rpc error: code = NotFound desc = failed to start sandbox \"e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:40.114429 6408 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\" with CreatePodSandboxError: \"Failed to create sandbox for pod \\\"priv-appliance-prometheus-node-exporter-qtwlg_default(48d05fd1-9744-41fd-83cb-f32cb8b2ed30)\\\": rpc error: code = NotFound desc = failed to start sandbox \\\"e913c4987fd7a1581939f1cacb2bc677dc058db2d5aa835d0962261f2dfd4f0b\\\": failed to get sandbox image \\\"rancher/mirrored-pause:3.6\\\": failed to pull image \\\"rancher/mirrored-pause:3.6\\\": failed to pull and unpack image \\\"docker.io/rancher/mirrored-pause:3.6\\\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found\"" pod="default/priv-appliance-prometheus-node-exporter-qtwlg" podUID="48d05fd1-9744-41fd-83cb-f32cb8b2ed30" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.235519863Z" level=debug msg="remove ingest" ref="manifest-sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.237817189Z" level=debug msg="schedule content cleanup" Mar 13 10:42:40 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:40.237849535Z" level=warning msg="content garbage collection failed" error="lstat /mnt/rancher/k3s/agent/containerd/io.containerd.content.v1.content/blobs: no such file or directory" Mar 13 10:42:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:41Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dbytes=0 entry" Mar 13 10:42:41 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:41Z" level=debug msg="cgroupv2 io stats: skipping over unmappable dios=0 entry" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.029884106Z" level=info msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.029997425Z" level=debug msg="Sandbox config metadata:{name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\"} hostname:\"priv-appliance-monitor-status-zm4mj\" log_directory:\"/var/log/pods/default_priv-appliance-monitor-status-zm4mj_4f679831-c7be-46a9-bab6-ea408cd29c72\" dns_config:{servers:\"10.43.0.10\" searches:\"default.svc.cluster.local\" searches:\"svc.cluster.local\" searches:\"cluster.local\" searches:\"us-west-2.compute.internal\" options:\"ndots:5\"} labels:{key:\"app\" value:\"monitor-status\"} labels:{key:\"app.securiti.ai/service-name\" value:\"status-monitor\"} labels:{key:\"controller-revision-hash\" value:\"7ccfc4d8ff\"} labels:{key:\"io.kubernetes.pod.name\" value:\"priv-appliance-monitor-status-zm4mj\"} labels:{key:\"io.kubernetes.pod.namespace\" value:\"default\"} labels:{key:\"io.kubernetes.pod.uid\" value:\"4f679831-c7be-46a9-bab6-ea408cd29c72\"} labels:{key:\"pod-template-generation\" value:\"1\"} labels:{key:\"release\" value:\"priv-appliance\"} annotations:{key:\"kubernetes.io/config.seen\" value:\"2026-03-13T10:36:00.014013728Z\"} annotations:{key:\"kubernetes.io/config.source\" value:\"api\"} linux:{cgroup_parent:\"/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod4f679831_c7be_46a9_bab6_ea408cd29c72.slice\" security_context:{namespace_options:{pid:CONTAINER userns_options:{mode:NODE}} seccomp:{}} overhead:{} resources:{cpu_period:100000 cpu_quota:100000 cpu_shares:102 memory_limit_in_bytes:786432000 unified:{key:\"memory.oom.group\" value:\"1\"}}}" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.030037604Z" level=debug msg="generated id for sandbox name \"priv-appliance-monitor-status-zm4mj_default_4f679831-c7be-46a9-bab6-ea408cd29c72_0\"" podsandboxid=21977e6f8667109d6499b6d3f759349ef6dd1ad0720ffc75e5c71a243d14f9e8 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.037971740Z" level=debug msg="begin cni setup" podsandboxid=21977e6f8667109d6499b6d3f759349ef6dd1ad0720ffc75e5c71a243d14f9e8 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: map[string]interface {}{"cniVersion":"1.0.0", "forceAddress":true, "hairpinMode":true, "ipMasq":false, "ipam":map[string]interface {}{"ranges":[][]map[string]interface {}{[]map[string]interface {}{map[string]interface {}{"subnet":"10.42.1.0/24"}}}, "routes":[]types.Route{types.Route{Dst:net.IPNet{IP:net.IP{0xa, 0x2a, 0x0, 0x0}, Mask:net.IPMask{0xff, 0xff, 0x0, 0x0}}, GW:net.IP(nil), MTU:0, AdvMSS:0, Priority:0, Table:(*int)(nil), Scope:(*int)(nil)}}, "type":"host-local"}, "isDefaultGateway":true, "isGateway":true, "mtu":(*uint)(0xc000010c60), "name":"cbr0", "type":"bridge"} Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: delegateAdd: netconf sent to delegate plugin: Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: {"cniVersion":"1.0.0","forceAddress":true,"hairpinMode":true,"ipMasq":false,"ipam":{"ranges":[[{"subnet":"10.42.1.0/24"}]],"routes":[{"dst":"10.42.0.0/16"}],"type":"host-local"},"isDefaultGateway":true,"isGateway":true,"mtu":8951,"name":"cbr0","type":"bridge"}time="2026-03-13T10:42:42.055037972Z" level=debug msg="cni result: {\"Interfaces\":{\"cni0\":{\"IPConfigs\":null,\"Mac\":\"5a:c0:94:53:e9:b9\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"},\"eth0\":{\"IPConfigs\":[{\"IP\":\"10.42.1.64\",\"Gateway\":\"10.42.1.1\"}],\"Mac\":\"52:a8:6f:a1:6f:cc\",\"Sandbox\":\"/var/run/netns/cni-70c12085-8760-a081-826c-30f0a53f8519\",\"PciID\":\"\",\"SocketPath\":\"\"},\"lo\":{\"IPConfigs\":[{\"IP\":\"127.0.0.1\",\"Gateway\":\"\"},{\"IP\":\"::1\",\"Gateway\":\"\"}],\"Mac\":\"00:00:00:00:00:00\",\"Sandbox\":\"/var/run/netns/cni-70c12085-8760-a081-826c-30f0a53f8519\",\"PciID\":\"\",\"SocketPath\":\"\"},\"veth7c2a8251\":{\"IPConfigs\":null,\"Mac\":\"f2:7e:1b:f1:5e:d6\",\"Sandbox\":\"\",\"PciID\":\"\",\"SocketPath\":\"\"}},\"DNS\":[{},{}],\"Routes\":[{\"dst\":\"10.42.0.0/16\"},{\"dst\":\"0.0.0.0/0\",\"gw\":\"10.42.1.1\"}]}" podsandboxid=21977e6f8667109d6499b6d3f759349ef6dd1ad0720ffc75e5c71a243d14f9e8 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.057217201Z" level=debug msg="PullImage using normalized image ref: \"docker.io/rancher/mirrored-pause:3.6\"" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.057235316Z" level=debug msg="PullImage \"docker.io/rancher/mirrored-pause:3.6\" with snapshotter overlayfs using client.Pull()" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.059135275Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.059341246Z" level=debug msg=resolving host="127.0.0.1:6443" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.059352642Z" level=debug msg="do request" host="127.0.0.1:6443" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=HEAD url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreig5lsn4dh3jstov5pvfqvqqjhjp2yq26zry5rk2maeqqxt5efsgee", "mh": "bciqn2xe3ygpwtfg5l27klblbasos7vrbv5tdr3cvuyajbbph2ilemii"} Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 1 provider entries Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:584 got provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:586 using provider: {QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt: [/ip4/10.151.22.85/tcp/5000]} Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.061Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42Z" level=info handler=mirror ip=127.0.0.1 latency=2.33173ms logger=spegel method=HEAD path=/v2/rancher/mirrored-pause/manifests/3.6 registry=docker.io status=200 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.063374722Z" level=debug msg="fetch response received" host="127.0.0.1:6443" response.header.cache-control="no-cache, private" response.header.content-length=399 response.header.content-type=application/vnd.oci.image.manifest.v1+json response.header.date="Fri, 13 Mar 2026 10:42:42 GMT" response.header.docker-content-digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" response.status="200 OK" url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/3.6?ns=docker.io" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.063412648Z" level=debug msg=resolved desc.digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" host="127.0.0.1:6443" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.063440906Z" level=debug msg="loading host directory" dir=/mnt/rancher/k3s/agent/etc/containerd/certs.d/docker.io Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.063653871Z" level=debug msg=fetch digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.065717855Z" level=debug msg="fetching layer" chunk_size=0 digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" initial_parallelism=3 mediatype=application/vnd.oci.image.manifest.v1+json offset=0 parallelism=1 size=399 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.065737203Z" level=debug msg="do request" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json request.header.accept="application/vnd.oci.image.manifest.v1+json, */*" request.header.accept-encoding="zstd;q=1.0, gzip;q=0.8, deflate;q=0.5" request.header.user-agent=containerd/v2.1.5-k3s1 request.method=GET size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.067Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:506 finding providers {"cid": "bafkreibyxxlewz3hvobjgxst7vgdgsp55f275mx3kmjwdwbasqcnzohd74", "mh": "bciqdrpowjntwpk4csnpfh7kmgne732lv72zpwuytmhmcbfae3s4oh7y"} Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:579 0 provider entries Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/routing.go:606 got closer peers: 0 [] Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: 2026-03-13T10:42:42.068Z DEBUG dht go-libp2p-kad-dht@v0.36.0/dht.go:712 peer found {"peer": "QmQJmcFMYMZv8FzFQVme6K2fyBpYnSFjRpWL4ZiaXkJCqt"} Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42Z" level=error error="MANIFEST_UNKNOWN could not find peer for sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" handler=mirror ip=127.0.0.1 latency=1.126797ms logger=spegel method=GET path="/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" registry=docker.io status=404 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.068520739Z" level=debug msg="fetch response received" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" mediatype=application/vnd.oci.image.manifest.v1+json response.header.cache-control="no-cache, private" response.header.content-length=172 response.header.content-type=application/json response.header.date="Fri, 13 Mar 2026 10:42:42 GMT" response.status="404 Not Found" size=399 url="https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.068558930Z" level=debug msg="fetch failed" digest="sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95" error="content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" mediatype=application/vnd.oci.image.manifest.v1+json size=399 Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.072417718Z" level=info msg="stop pulling image docker.io/rancher/mirrored-pause:3.6: active requests=0, bytes read=0" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: time="2026-03-13T10:42:42.107331311Z" level=error msg="RunPodSandbox for name:\"priv-appliance-monitor-status-zm4mj\" uid:\"4f679831-c7be-46a9-bab6-ea408cd29c72\" namespace:\"default\" failed, error" error="rpc error: code = NotFound desc = failed to start sandbox \"21977e6f8667109d6499b6d3f759349ef6dd1ad0720ffc75e5c71a243d14f9e8\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:42.107475 6408 log.go:32] "RunPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = failed to start sandbox \"21977e6f8667109d6499b6d3f759349ef6dd1ad0720ffc75e5c71a243d14f9e8\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" Mar 13 10:42:42 ip-10-151-21-17.us-west-2.compute.internal k3s[6408]: E0313 10:42:42.107506 6408 kuberuntime_sandbox.go:71] "Failed to create sandbox for pod" err="rpc error: code = NotFound desc = failed to start sandbox \"21977e6f8667109d6499b6d3f759349ef6dd1ad0720ffc75e5c71a243d14f9e8\": failed to get sandbox image \"rancher/mirrored-pause:3.6\": failed to pull image \"rancher/mirrored-pause:3.6\": failed to pull and unpack image \"docker.io/rancher/mirrored-pause:3.6\": failed to copy: httpReadSeeker: failed open: content at https://127.0.0.1:6443/v2/rancher/mirrored-pause/manifests/sha256:16974531848218d24822bf606be022d030ab8c9b05b2ecf11076c4c1c6885c95?ns=docker.io not found: not found" pod="default/priv-appliance-monitor-status-zm4mj"