Ingress Nginx 代理高级配置与性能优化

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-configuration-registry-proxy
  namespace: ingress-nginx
data:
  # 基础性能设置
  worker-processes: "auto"
  worker-connections: "65536"
  worker-rlimit-nofile: "65536"
  
  # 客户端连接优化
  keepalive-timeout: "65"
  keepalive-requests: "10000"
  client-header-timeout: "60"
  client-body-timeout: "60"
  
  # 代理优化
  proxy-connect-timeout: "600"
  proxy-send-timeout: "600"
  proxy-read-timeout: "600"
  proxy-buffer-size: "16k"
  proxy-buffers-number: "8"
  proxy-max-temp-file-size: "1024m"
  proxy-body-size: "0"
  
  # 缓存配置
  proxy-buffering: "on"
  proxy-cache-path: "/var/cache/nginx/registry levels=1:2 keys_zone=registry_cache:100m max_size=10g inactive=60m use_temp_path=off"
  proxy-cache: "registry_cache"
  proxy-cache-valid: "200 302 10m"
  proxy-cache-valid: "404 1m"
  proxy-cache-key: "$scheme$proxy_host$request_uri"
  
  # Gzip 压缩
  use-gzip: "true"
  gzip-level: "6"
  gzip-min-length: "1024"
  gzip-types: |
    application/json
    application/vnd.docker.distribution.manifest.v1+json
    application/vnd.docker.distribution.manifest.v2+json
    application/vnd.docker.distribution.manifest.list.v2+json
    application/vnd.docker.distribution.config.v2+json
  
  # 日志格式
  log-format-upstream: |
    $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: docker-hub-proxy-advanced
  namespace: default
  annotations:
    # 基本代理配置
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "600"
    
    # 缓冲区配置
    nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
    nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
    nginx.ingress.kubernetes.io/proxy-max-temp-file-size: "1024m"
    nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
    
    # 连接池配置
    nginx.ingress.kubernetes.io/upstream-keepalive-connections: "320"
    nginx.ingress.kubernetes.io/upstream-keepalive-requests: "10000"
    nginx.ingress.kubernetes.io/upstream-keepalive-timeout: "60"
    
    # 重试和超时配置
    nginx.ingress.kubernetes.io/proxy-next-upstream: "error timeout invalid_header http_500 http_502 http_503 http_504"
    nginx.ingress.kubernetes.io/proxy-next-upstream-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-next-upstream-tries: "3"
    
    # 自定义配置片段
    nginx.ingress.kubernetes.io/configuration-snippet: |
      # 缓存配置
      proxy_cache registry_cache;
      proxy_cache_valid 200 302 10m;
      proxy_cache_valid 404 1m;
      proxy_cache_bypass $http_pragma;
      proxy_cache_revalidate on;
      
      # 添加缓存控制头
      add_header X-Cache-Status $upstream_cache_status;
      add_header Cache-Control "public, max-age=600";
      
      # 安全头
      add_header X-Content-Type-Options nosniff;
      add_header X-Frame-Options DENY;
      add_header X-XSS-Protection "1; mode=block";
      
      # Docker Registry 特定的头
      proxy_set_header Docker-Distribution-Api-Version registry/2.0;
      
      # 日志记录
      access_log /var/log/nginx/registry_access.log;
      error_log /var/log/nginx/registry_error.log info;
      
    # 服务器配置片段
    nginx.ingress.kubernetes.io/server-snippet: |
      # 限流配置
      limit_req_zone $binary_remote_addr zone=registry_limit:10m rate=10r/s;
      limit_req zone=registry_limit burst=20 nodelay;
      
      # 连接数限制
      limit_conn_zone $binary_remote_addr zone=addr:10m;
      limit_conn addr 100;

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

apiVersion: v1
kind: Service
metadata:
  name: docker-hub-registry-primary
  namespace: default
spec:
  type: ExternalName
  externalName: registry-1.docker.io
---
apiVersion: v1
kind: Service
metadata:
  name: docker-hub-registry-mirror
  namespace: default
spec:
  type: ExternalName
  externalName: mirror.gcr.io  # Google 的 Docker Hub 镜像
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: docker-hub-proxy-lb
  namespace: default
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
    nginx.ingress.kubernetes.io/configuration-snippet: |
      # 健康检查
      proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
      proxy_connect_timeout 2s;
      proxy_send_timeout 10s;
      proxy_read_timeout 10s;
      
spec:
  ingressClassName: nginx
  rules:
  - host: dockerhub-lb.yourdomain.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: docker-hub-registry-primary
            port:
              number: 443

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

apiVersion: v1
kind: Secret
metadata:
  name: registry-basic-auth
  namespace: default
type: Opaque
data:
  # 使用 htpasswd 生成: htpasswd -nb user password
  # echo -n 'user:$apr1$xyz$passwordhash' | base64
  auth: dXNlcjokYXByMSR4eXokcGFzc3dvcmRoYXNo
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: docker-hub-proxy-secure
  namespace: default
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    nginx.ingress.kubernetes.io/auth-type: basic
    nginx.ingress.kubernetes.io/auth-secret: registry-basic-auth
    nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - Docker Registry"
    nginx.ingress.kubernetes.io/configuration-snippet: |
      # IP 白名单
      allow 10.0.0.0/8;
      allow 172.16.0.0/12;
      allow 192.168.0.0/16;
      deny all;
      
      # 额外的安全头
      add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
      add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
      
spec:
  ingressClassName: nginx
  rules:
  - host: secure-dockerhub.yourdomain.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: docker-hub-registry
            port:
              number: 443
  tls:
  - hosts:
    - secure-dockerhub.yourdomain.com
    secretName: secure-dockerhub-tls

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-configuration-metrics
  namespace: ingress-nginx
data:
  enable-metrics: "true"
  metrics-port: "10254"
  metrics-per-host: "true"
  
  # 自定义指标标签
  metrics-labels: "service=registry-proxy,environment=production"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: ingress-nginx-registry-proxy
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/component: controller
  endpoints:
  - port: metrics
    interval: 30s
    path: /metrics
    honorLabels: true
    relabelings:
    - sourceLabels: [__meta_kubernetes_service_name]
      targetLabel: service
    - sourceLabels: [__meta_kubernetes_namespace]
      targetLabel: namespace

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

apiVersion: apps/v1
kind: Deployment
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  template:
    spec:
      containers:
      - name: controller
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 2000m
            memory: 4Gi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

livenessProbe:
  httpGet:
    path: /healthz
    port: 10254
    scheme: HTTP
  initialDelaySeconds: 10
  periodSeconds: 10
  timeoutSeconds: 1
  successThreshold: 1
  failureThreshold: 3

readinessProbe:
  httpGet:
    path: /healthz
    port: 10254
    scheme: HTTP
  initialDelaySeconds: 10
  periodSeconds: 10
  timeoutSeconds: 1
  successThreshold: 1
  failureThreshold: 3

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: ingress-nginx-registry-proxy
  namespace: ingress-nginx
spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/component: controller
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - namespaceSelector: {}
    ports:
    - protocol: TCP
      port: 80
    - protocol: TCP
      port: 443
    - protocol: TCP
      port: 10254
  egress:
  - to: []
    ports:
    - protocol: TCP
      port: 443  # HTTPS 到外部注册表
    - protocol: TCP
      port: 53   # DNS
    - protocol: UDP
      port: 53   # DNS

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# 测试 Docker Hub 代理
docker pull dockerhub.yourdomain.com/library/nginx:latest

# 测试 GCR 代理
docker pull gcr.yourdomain.com/google-containers/pause:3.9

# 测试 Quay.io 代理
docker pull quay.yourdomain.com/coreos/etcd:v3.5.9

# 查看 Nginx 日志
kubectl logs -n ingress-nginx deployment/ingress-nginx-controller | grep registry

# 查看指标（如果启用了 Prometheus）
curl http://<ingress-controller-ip>:10254/metrics | grep nginx