Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions docs/NetworkPolicy.md
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,30 @@ operand namespace (e.g. `tekton-pipelines` or `openshift-pipelines`).
|---|---|---|
| DNS port | 53 | 5353 |
| DNS namespace | `kube-system` | `openshift-dns` |
| DNS link-local ipBlocks | `169.254.169.254/32`, `169.254.169.253/32`, `169.254.20.10/32` | none |
| Prometheus namespace label | `kubernetes.io/metadata.name: monitoring` | `openshift.io/cluster-monitoring: "true"` |

On Kubernetes, every DNS egress rule also allows UDP+TCP 53 to the link-local
resolvers above. Some clusters point pods at a resolver that is not the kube-dns
Service: GKE clusters using [Cloud DNS][clouddns] write `169.254.169.254` (the
node-local metadata server running the Cloud DNS data plane) into every pod's
resolv.conf, and clusters with [NodeLocal DNSCache][nodelocal] (default on GKE
Autopilot) use `169.254.20.10`. The AWS VPC resolver ([Route 53 Resolver][awsdns])
listens at `169.254.169.253`; pods query it directly only with non-default DNS
configuration (default `ClusterFirst` pods use the CoreDNS Service IP and never
send to link-local), but it is allowed for parity so custom DNS setups work
without overrides. These are host-network endpoints that no
`podSelector`/`namespaceSelector` can match, so they are allowed via `ipBlock`
peers instead. The peers are port-scoped to DNS (53) only — other ports on the
metadata server, such as IMDS on 80, remain blocked. On clusters not using
these resolvers the peers never match. If you run NodeLocal DNSCache with a
non-default `localip`, allow that address by overriding the policy by name
(see [Overriding a policy](#overriding-a-policy)).

[clouddns]: https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns
[nodelocal]: https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/
[awsdns]: https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html

## Disabling

```yaml
Expand Down
31 changes: 20 additions & 11 deletions pkg/reconciler/common/networkpolicy/networkpolicy.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,27 +99,36 @@ func DefaultDenyPolicy(name string, podSelector metav1.LabelSelector) networking
}

// DNSEgressRule allows egress to DNS resolver pods on UDP and TCP using the
// platform-specific DNS port (53 on Kubernetes, 5353 on OpenShift).
// platform-specific DNS port (53 on Kubernetes, 5353 on OpenShift). On
// Kubernetes it also allows the same ports to link-local resolvers (GKE Cloud
// DNS, NodeLocal DNSCache) via ipBlock — those are host-network endpoints that
// no pod selector can match.
func DNSEgressRule(p PlatformParams) networkingv1.NetworkPolicyEgressRule {
udp := corev1.ProtocolUDP
tcp := corev1.ProtocolTCP
udpPort := intstr.FromInt32(p.DNSPort)
tcpPort := intstr.FromInt32(p.DNSPort)
to := []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"kubernetes.io/metadata.name": p.DNSResolverNamespace},
},
PodSelector: &metav1.LabelSelector{
MatchLabels: p.DNSResolverPodLabel,
},
},
}
for _, cidr := range p.DNSResolverIPBlocks {
to = append(to, networkingv1.NetworkPolicyPeer{
IPBlock: &networkingv1.IPBlock{CIDR: cidr},
})
}
return networkingv1.NetworkPolicyEgressRule{
Ports: []networkingv1.NetworkPolicyPort{
{Protocol: &udp, Port: &udpPort},
{Protocol: &tcp, Port: &tcpPort},
},
To: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"kubernetes.io/metadata.name": p.DNSResolverNamespace},
},
PodSelector: &metav1.LabelSelector{
MatchLabels: p.DNSResolverPodLabel,
},
},
},
To: to,
}
}

Expand Down
25 changes: 21 additions & 4 deletions pkg/reconciler/common/networkpolicy/networkpolicy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,20 +211,37 @@ func TestDNSEgressRule_Kubernetes(t *testing.T) {
t.Errorf("expected DNS port 53 for Kubernetes, got %d", p.Port.IntVal)
}
}
if len(rule.To) != 1 || rule.To[0].NamespaceSelector == nil {
t.Fatalf("expected 1 To with NamespaceSelector, got %v", rule.To)
if len(rule.To) != 4 || rule.To[0].NamespaceSelector == nil {
t.Fatalf("expected 4 To peers (kube-dns selector + 3 ipBlocks), got %v", rule.To)
}
nsLabels := rule.To[0].NamespaceSelector.MatchLabels
if nsLabels["kubernetes.io/metadata.name"] != "kube-system" {
t.Errorf("expected kube-system namespace selector, got %v", nsLabels)
}
wantCIDRs := map[string]bool{"169.254.169.254/32": false, "169.254.169.253/32": false, "169.254.20.10/32": false}
for _, peer := range rule.To[1:] {
if peer.IPBlock == nil {
t.Errorf("expected ipBlock peer, got %v", peer)
continue
}
if _, ok := wantCIDRs[peer.IPBlock.CIDR]; !ok {
t.Errorf("unexpected ipBlock CIDR %s", peer.IPBlock.CIDR)
continue
}
wantCIDRs[peer.IPBlock.CIDR] = true
}
for cidr, seen := range wantCIDRs {
if !seen {
t.Errorf("missing ipBlock peer for %s", cidr)
}
}
}

func TestDNSEgressRule_OpenShift(t *testing.T) {
params := networkpolicy.OpenShiftPlatformDefaults()
rule := networkpolicy.DNSEgressRule(params)
if len(rule.To) == 0 || rule.To[0].NamespaceSelector == nil {
t.Fatalf("expected 1 To with NamespaceSelector, got %v", rule.To)
if len(rule.To) != 1 || rule.To[0].NamespaceSelector == nil {
t.Fatalf("expected exactly 1 To peer with NamespaceSelector (no ipBlocks on OpenShift), got %v", rule.To)
}
nsLabels := rule.To[0].NamespaceSelector.MatchLabels
if nsLabels["kubernetes.io/metadata.name"] != "openshift-dns" {
Expand Down
23 changes: 19 additions & 4 deletions pkg/reconciler/common/networkpolicy/platform.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,15 @@ package networkpolicy
// PlatformParams holds platform-specific values for building default NetworkPolicy rules.
// It is an internal type — it never appears in CRD API fields.
type PlatformParams struct {
DNSResolverNamespace string
DNSResolverPodLabel map[string]string
DNSResolverNamespace string
DNSResolverPodLabel map[string]string
// DNSResolverIPBlocks are link-local resolver IPs kubelet may write into pod
// resolv.conf instead of the kube-dns Service: GKE Cloud DNS (169.254.169.254),
// NodeLocal DNSCache (169.254.20.10, GKE Autopilot default) and the AWS VPC
// resolver (169.254.169.253, reached directly only with non-default DNS
// configuration). They are host-network endpoints no pod selector can match,
// so only ipBlock peers can allow them. Empty on OpenShift.
DNSResolverIPBlocks []string
PrometheusNamespaceLabel map[string]string
// DNSPort is the DNS resolver pod port. Kubernetes CoreDNS uses 53; OpenShift DNS
// uses 5353 (OVN-K8s enforces NetworkPolicy after DNAT, so pod port applies).
Expand All @@ -30,8 +37,16 @@ type PlatformParams struct {
// KubernetesPlatformDefaults returns PlatformParams for vanilla Kubernetes.
func KubernetesPlatformDefaults() PlatformParams {
return PlatformParams{
DNSResolverNamespace: "kube-system",
DNSResolverPodLabel: map[string]string{"k8s-app": "kube-dns"},
DNSResolverNamespace: "kube-system",
DNSResolverPodLabel: map[string]string{"k8s-app": "kube-dns"},
DNSResolverIPBlocks: []string{
// 169.254.169.254/32: GKE Cloud DNS forwarder (link-local, host-network).
"169.254.169.254/32",
// 169.254.169.253/32: AWS VPC resolver (Route 53 Resolver, link-local).
"169.254.169.253/32",
// 169.254.20.10/32: NodeLocal DNSCache default localip (GKE Autopilot).
"169.254.20.10/32",
},
PrometheusNamespaceLabel: map[string]string{"kubernetes.io/metadata.name": "monitoring"},
DNSPort: 53,
}
Expand Down
Loading