I’m trying to use AL9 for Kubernetes node with Rancher, but there seems to be some sort of network routing issue with Flannel and kube-dns after deploying the server. I’ve verified that Docker installs and runs without any DNS issue. Firewalld is disabled and I can see that iptables rules are being created. However, any attempts to perform DNS against the kube-dns coredns instance fails.
[root@dev-tor-worker-05 ~]# dig @10.43.0.10 google.com
; <<>> DiG 9.16.23-RH <<>> @10.43.0.10 google.com
; (1 server found)
;; global options: +cmd
;; connection timed out; no servers could be reached
[root@dev-tor-worker-05 ~]# iptables -L
Chain INPUT (policy ACCEPT)
target prot opt source destination
cali-INPUT all -- anywhere anywhere /* cali:Cz_u1IQiXIMmKD4c */
KUBE-FIREWALL all -- anywhere anywhere
KUBE-NODEPORTS all -- anywhere anywhere /* kubernetes health check service ports */
KUBE-EXTERNAL-SERVICES all -- anywhere anywhere ctstate NEW /* kubernetes externally-visible service portals */
Chain FORWARD (policy DROP)
target prot opt source destination
cali-FORWARD all -- anywhere anywhere /* cali:wUHhoiAYhphO9Mso */
KUBE-FORWARD all -- anywhere anywhere /* kubernetes forwarding rules */
KUBE-SERVICES all -- anywhere anywhere ctstate NEW /* kubernetes service portals */
KUBE-EXTERNAL-SERVICES all -- anywhere anywhere ctstate NEW /* kubernetes externally-visible service portals */
DOCKER-USER all -- anywhere anywhere
DOCKER-ISOLATION-STAGE-1 all -- anywhere anywhere
ACCEPT all -- anywhere anywhere ctstate RELATED,ESTABLISHED
DOCKER all -- anywhere anywhere
ACCEPT all -- anywhere anywhere
ACCEPT all -- anywhere anywhere
ACCEPT all -- 10.42.0.0/16 anywhere /* flanneld forward */
ACCEPT all -- anywhere 10.42.0.0/16 /* flanneld forward */
ACCEPT all -- anywhere anywhere /* cali:S93hcgKJrXEqnTfs */ /* Policy explicitly accepted packet. */ mark match 0x10000/0x10000
MARK all -- anywhere anywhere /* cali:mp77cMpurHhyjLrM */ MARK or 0x10000
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
cali-OUTPUT all -- anywhere anywhere /* cali:tVnHkvAo15HuiPy0 */
KUBE-FIREWALL all -- anywhere anywhere
KUBE-SERVICES all -- anywhere anywhere ctstate NEW /* kubernetes service portals */
Chain DOCKER (1 references)
target prot opt source destination
Chain DOCKER-ISOLATION-STAGE-1 (1 references)
target prot opt source destination
DOCKER-ISOLATION-STAGE-2 all -- anywhere anywhere
RETURN all -- anywhere anywhere
Chain DOCKER-ISOLATION-STAGE-2 (1 references)
target prot opt source destination
DROP all -- anywhere anywhere
RETURN all -- anywhere anywhere
Chain DOCKER-USER (1 references)
target prot opt source destination
RETURN all -- anywhere anywhere
Chain KUBE-EXTERNAL-SERVICES (2 references)
target prot opt source destination
Chain KUBE-FIREWALL (2 references)
target prot opt source destination
DROP all -- anywhere anywhere /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000
DROP all -- !127.0.0.0/8 127.0.0.0/8 /* block incoming localnet connections */ ! ctstate RELATED,ESTABLISHED,DNAT
Chain KUBE-FORWARD (1 references)
target prot opt source destination
DROP all -- anywhere anywhere ctstate INVALID
ACCEPT all -- anywhere anywhere /* kubernetes forwarding rules */ mark match 0x4000/0x4000
ACCEPT all -- anywhere anywhere /* kubernetes forwarding conntrack rule */ ctstate RELATED,ESTABLISHED
Chain KUBE-KUBELET-CANARY (0 references)
target prot opt source destination
Chain KUBE-NODEPORTS (1 references)
target prot opt source destination
Chain KUBE-PROXY-CANARY (0 references)
target prot opt source destination
Chain KUBE-SERVICES (2 references)
target prot opt source destination
Chain cali-FORWARD (1 references)
target prot opt source destination
MARK all -- anywhere anywhere /* cali:vjrMJCRpqwy5oRoX */ MARK and 0xfff1ffff
cali-from-hep-forward all -- anywhere anywhere /* cali:A_sPAO0mcxbT9mOV */ mark match 0x0/0x10000
cali-from-wl-dispatch all -- anywhere anywhere /* cali:8ZoYfO5HKXWbB3pk */
cali-to-wl-dispatch all -- anywhere anywhere /* cali:jdEuaPBe14V2hutn */
cali-to-hep-forward all -- anywhere anywhere /* cali:12bc6HljsMKsmfr- */
cali-cidr-block all -- anywhere anywhere /* cali:NOSxoaGx8OIstr1z */
Chain cali-INPUT (1 references)
target prot opt source destination
cali-wl-to-host all -- anywhere anywhere [goto] /* cali:FewJpBykm9iJ-YNH */
ACCEPT all -- anywhere anywhere /* cali:hder3ARWznqqv8Va */ mark match 0x10000/0x10000
MARK all -- anywhere anywhere /* cali:xgOu2uJft6H9oDGF */ MARK and 0xfff0ffff
cali-from-host-endpoint all -- anywhere anywhere /* cali:_-d-qojMfHM6NwBo */
ACCEPT all -- anywhere anywhere /* cali:LqmE76MP94lZTGhA */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000
Chain cali-OUTPUT (1 references)
target prot opt source destination
ACCEPT all -- anywhere anywhere /* cali:Mq1_rAdXXH3YkrzW */ mark match 0x10000/0x10000
RETURN all -- anywhere anywhere /* cali:69FkRTJDvD5Vu6Vl */
MARK all -- anywhere anywhere /* cali:Fskumj4SGQtDV6GC */ MARK and 0xfff0ffff
cali-to-host-endpoint all -- anywhere anywhere /* cali:1F4VWEsQu0QbRwKf */ ! ctstate DNAT
ACCEPT all -- anywhere anywhere /* cali:m8Eqm15x1MjD24LD */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000
Chain cali-cidr-block (1 references)
target prot opt source destination
Chain cali-from-hep-forward (1 references)
target prot opt source destination
Chain cali-from-host-endpoint (1 references)
target prot opt source destination
Chain cali-from-wl-dispatch (2 references)
target prot opt source destination
cali-fw-calid93d8ec1b4d all -- anywhere anywhere [goto] /* cali:7zjGfzIG1XomskZG */
DROP all -- anywhere anywhere /* cali:mrQfhdcZIsJt1-5D */ /* Unknown interface */
Chain cali-fw-calid93d8ec1b4d (1 references)
target prot opt source destination
ACCEPT all -- anywhere anywhere /* cali:_yHoVHc5xIZXYXiu */ ctstate RELATED,ESTABLISHED
DROP all -- anywhere anywhere /* cali:EvoSL_tBVx4v-BLq */ ctstate INVALID
MARK all -- anywhere anywhere /* cali:eypXXjb8cgbDRK56 */ MARK and 0xfffeffff
DROP udp -- anywhere anywhere /* cali:d9J8iZ-KzV-Dt9o0 */ /* Drop VXLAN encapped packets originating in workloads */ multiport dports vxlan
DROP ipv4 -- anywhere anywhere /* cali:uEuQU3HwvzGqBz84 */ /* Drop IPinIP encapped packets originating in workloads */
cali-pro-kns.ingress-nginx all -- anywhere anywhere /* cali:gY3XJODLnRxDgabQ */
RETURN all -- anywhere anywhere /* cali:HKBW_bI1HrByXyQX */ /* Return if profile accepted */ mark match 0x10000/0x10000
cali-pro-_WuAV8wMhwxuQO3vuFE all -- anywhere anywhere /* cali:Ew9kjFuD4DN9qQb_ */
RETURN all -- anywhere anywhere /* cali:2nim1axEJtCXC8Y8 */ /* Return if profile accepted */ mark match 0x10000/0x10000
DROP all -- anywhere anywhere /* cali:LLS3iyx-cfR_E8n- */ /* Drop if no profiles matched */
Chain cali-pri-_WuAV8wMhwxuQO3vuFE (1 references)
target prot opt source destination
all -- anywhere anywhere /* cali:v_wvZWiaEOjMs3ly */ /* Profile ksa.ingress-nginx.ingress-nginx ingress */
Chain cali-pri-kns.ingress-nginx (1 references)
target prot opt source destination
MARK all -- anywhere anywhere /* cali:gbVRyyjiyhHkpyKH */ /* Profile kns.ingress-nginx ingress */ MARK or 0x10000
RETURN all -- anywhere anywhere /* cali:h9qOXJdthg_rMYN5 */ mark match 0x10000/0x10000
Chain cali-pro-_WuAV8wMhwxuQO3vuFE (1 references)
target prot opt source destination
all -- anywhere anywhere /* cali:EQD2q2n1iv7oL_7a */ /* Profile ksa.ingress-nginx.ingress-nginx egress */
Chain cali-pro-kns.ingress-nginx (1 references)
target prot opt source destination
MARK all -- anywhere anywhere /* cali:hHsDqH-8IwubWuZc */ /* Profile kns.ingress-nginx egress */ MARK or 0x10000
RETURN all -- anywhere anywhere /* cali:ewSE6gCHQIsXlBSR */ mark match 0x10000/0x10000
Chain cali-to-hep-forward (1 references)
target prot opt source destination
Chain cali-to-host-endpoint (1 references)
target prot opt source destination
Chain cali-to-wl-dispatch (1 references)
target prot opt source destination
cali-tw-calid93d8ec1b4d all -- anywhere anywhere [goto] /* cali:3hlx5vxmbL6UCuQZ */
DROP all -- anywhere anywhere /* cali:jg2s64i9D1Ay1qus */ /* Unknown interface */
Chain cali-tw-calid93d8ec1b4d (1 references)
target prot opt source destination
ACCEPT all -- anywhere anywhere /* cali:JareqNq3_h-YosV4 */ ctstate RELATED,ESTABLISHED
DROP all -- anywhere anywhere /* cali:jaVpSeU71RSwLtws */ ctstate INVALID
MARK all -- anywhere anywhere /* cali:RSoyTp7JDbpIyvtc */ MARK and 0xfffeffff
cali-pri-kns.ingress-nginx all -- anywhere anywhere /* cali:shnDkTR9J6llLs21 */
RETURN all -- anywhere anywhere /* cali:UOaY8hqQ5m_jcuDX */ /* Return if profile accepted */ mark match 0x10000/0x10000
cali-pri-_WuAV8wMhwxuQO3vuFE all -- anywhere anywhere /* cali:GFaXXZxYuju-Qk0y */
RETURN all -- anywhere anywhere /* cali:L5hvK17jFOLW-dY3 */ /* Return if profile accepted */ mark match 0x10000/0x10000
DROP all -- anywhere anywhere /* cali:vFnc9LHOd4dinHs_ */ /* Drop if no profiles matched */
Chain cali-wl-to-host (1 references)
target prot opt source destination
cali-from-wl-dispatch all -- anywhere anywhere /* cali:Ee9Sbo10IpVujdIY */
ACCEPT all -- anywhere anywhere /* cali:nSZbcOoG1xPONxb8 */ /* Configured DefaultEndpointToHostAction */