Kubernetes: 08-gateway-api

# 安装标准版 gateway api CRD
kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.1/standard-install.yaml

# 范例
root@ip-172-31-18-198:/docker/nginx/data/site.d# kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.1/standard-install.yaml
customresourcedefinition.apiextensions.k8s.io/backendtlspolicies.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/gatewayclasses.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/gateways.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/grpcroutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/httproutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/listenersets.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/referencegrants.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/tcproutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/tlsroutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/udproutes.gateway.networking.k8s.io serverside-applied
validatingadmissionpolicy.admissionregistration.k8s.io/safe-upgrades.gateway.networking.k8s.io serverside-applied
validatingadmissionpolicybinding.admissionregistration.k8s.io/safe-upgrades.gateway.networking.k8s.io serverside-applied

root@ip-172-31-18-198:~# kubectl  get crd |grep gateway.networking.k8s.io
backendtlspolicies.gateway.networking.k8s.io    2026-08-26T09:10:31Z
gatewayclasses.gateway.networking.k8s.io        2026-08-26T09:10:31Z
gateways.gateway.networking.k8s.io              2026-08-26T09:10:31Z
grpcroutes.gateway.networking.k8s.io            2026-08-26T09:10:31Z
httproutes.gateway.networking.k8s.io            2026-08-26T09:10:31Z
listenersets.gateway.networking.k8s.io          2026-08-26T09:10:32Z
referencegrants.gateway.networking.k8s.io       2026-08-26T09:10:32Z
tcproutes.gateway.networking.k8s.io             2026-08-26T09:10:32Z
tlsroutes.gateway.networking.k8s.io             2026-08-26T09:10:33Z
udproutes.gateway.networking.k8s.io             2026-08-26T09:10:33Z

# 安装 NGINX Gateway Fabric CRDs
# https://github.com/nginx/nginx-gateway-fabric/tree/v2.6.7/deploy
kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/refs/tags/v2.6.7/deploy/crds.yaml
# 普通 kubectl apply 是客户端 apply：本地保存一份注解 kubectl.kubernetes.io/last-applied-configuration，用来对比变更。
# --server‑side 是把合并逻辑交给 **k8s APIServer 服务端处理**，不在客户端存大段 last‑applied 注解。

# 范例
root@ip-172-31-18-198:~# kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/refs/tags/v2.6.7/deploy/crds.yaml
customresourcedefinition.apiextensions.k8s.io/authenticationfilters.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/clientsettingspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/nginxgateways.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/nginxproxies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/observabilitypolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/proxysettingspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/ratelimitpolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/snippetsfilters.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/snippetspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/upstreamsettingspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/wafpolicies.gateway.nginx.org serverside-applied

root@ip-172-31-18-198:~# kubectl  get crd |grep nginx
authenticationfilters.gateway.nginx.org         2026-08-26T09:37:50Z
clientsettingspolicies.gateway.nginx.org        2026-08-26T09:37:50Z
nginxgateways.gateway.nginx.org                 2026-08-26T09:37:50Z
nginxproxies.gateway.nginx.org                  2026-08-26T09:37:50Z
observabilitypolicies.gateway.nginx.org         2026-08-26T09:37:50Z # 可观测规则
proxysettingspolicies.gateway.nginx.org         2026-08-26T09:37:51Z # 代理设置规则
ratelimitpolicies.gateway.nginx.org             2026-08-26T09:37:51Z # 限速规则
snippetsfilters.gateway.nginx.org               2026-08-26T09:37:51Z
snippetspolicies.gateway.nginx.org              2026-08-26T09:37:51Z
upstreamsettingspolicies.gateway.nginx.org      2026-08-26T09:37:51Z
wafpolicies.gateway.nginx.org                   2026-08-26T09:37:51Z # 应用防火墙规则

# 
mkdir ngfdir
cd ngfdir/

# 下载 2.6.7 版本文件。拉取 OCI 格式 Helm chart，解压到本地
helm pull oci://ghcr.io/nginx/charts/nginx-gateway-fabric --version 2.6.7 --untar
cd nginx-gateway-fabric/

helm install ngf . \
  --create-namespace -n nginx-gateway \
  --set nginx.service.type=LoadBalancer \
  --set nginx.service.externalTrafficPolicy=Cluster \
  --set nginxGateway.snippetsFilters.enable=true

#  --set nginx.service.type=LoadBalancer 使用负载均衡器。会自动分配公网 LB
#  --set nginx.service.externalTrafficPolicy=Cluster 外部流量转发策略。
#    Cluster（这里配置的）：源 IP 会被 SNAT 改写为节点 IP；可以做负载均衡跨节点转发。
#    Local：保留真实客户端源 IP，但流量只能落到后端 Pod 所在节点。
#  --set nginxGateway.snippets.enable=true  NGF 的扩展能力，可以在 Gateway API 资源中嵌入原生 Nginx 配置片段（nginx.conf 片段）。同时启用 SnippetsFilter 和 SnippetsPolicy。

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get ns |grep gateway
nginx-gateway       Active   16m

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get pods -n nginx-gateway
NAME                                        READY   STATUS    RESTARTS   AGE
ngf-nginx-gateway-fabric-746f6d68fc-hspmf   1/1     Running   0          85s

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get svc -n nginx-gateway
NAME                       TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
ngf-nginx-gateway-fabric   ClusterIP   10.210.167.53   <none>        443/TCP   2m7s

# 卸载
# helm uninstall ngf -n nginx-gateway
# kubectl delete ns nginx-gateway --ignore-not-found

# 控制器 nginx
root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get gatewayclasses -n nginx-gateway
NAME    CONTROLLER                                   ACCEPTED   AGE
nginx   gateway.nginx.org/nginx-gateway-controller   True       2m55s

# nginxproxy 相当说configmap
root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get nginxproxy -n nginx-gateway
NAME               AGE
ngf-proxy-config   4m33s

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get nginxproxies -n nginx-gateway  -oyaml
apiVersion: v1
items:
- apiVersion: gateway.nginx.org/v1alpha2
  kind: NginxProxy
  metadata:
    labels:
      app.kubernetes.io/instance: ngf
      app.kubernetes.io/managed-by: Helm
      app.kubernetes.io/name: nginx-gateway-fabric
      app.kubernetes.io/version: 2.6.7
      helm.sh/chart: nginx-gateway-fabric-2.6.7
    name: ngf-proxy-config
    namespace: nginx-gateway
  spec:
    ipFamily: dual
    kubernetes:
      deployment:
        container:
          image:
            pullPolicy: IfNotPresent
            repository: ghcr.io/nginx/nginx-gateway-fabric/nginx
            tag: 2.6.7
        replicas: 1
      service:
        externalTrafficPolicy: Cluster
        type: LoadBalancer
kind: List
metadata:
  resourceVersion: ""

cat <<\EOF>> 01-gateway-test.yaml
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: nginx-gateway-test
  namespace: default
spec:
  gatewayClassName: nginx
  listeners:
  - name: http
    protocol: HTTP
    port: 80
    allowedRoutes:
      namespaces:
        from: All # 允许所有namespace的HTTPRoute绑定这个Gateway。默认为允许当前名称空间的规则
EOF

kubectl apply -f 01-gateway-test.yaml

# gateway 的地址就是 service 的地址
root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  get gateway
NAME                 CLASS   ADDRESS                                                                             PROGRAMMED   AGE
nginx-gateway-test   nginx   k8s-d-1.amazonaws.com   True         25m

root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  get svc
nginx-gateway-test-nginx       LoadBalancer   10.210.170.159   k8s-d-1.amazonaws.com   80:31363/TCP

kubectl -n default get svc nginx-gateway-test-nginx \
    -o jsonpath='class=[{.spec.loadBalancerClass}] ports={.spec.ports[*].port}{"\n"}'

# 生成证书
curl https://get.acme.sh | sh -s email=jasper@jasper.org


export CF_Token="cfut_xxxx"
export CF_Email="xxx@gmail.com"
acme.sh --set-default-ca --server letsencrypt
acme.sh --issue --dns dns_cf -d jasper.org -d *.jasper.org --dnssleep 120


# 给 gateway 加证书
kubectl -n default create secret tls jasper-tls --cert=jasper.org.cer --key=jasper.org.key

cat > gateway-public.yaml <<\EOF
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: gateway-public
  namespace: default
spec:
  gatewayClassName: nginx
  listeners:
  - allowedRoutes:
      namespaces:
        from: All
    name: http
    port: 80
    protocol: HTTP
  - name: https
    port: 443
    protocol: HTTPS
    hostname: "*.xxx.com"
    tls:
      mode: Terminate
      certificateRefs:
        - kind: Secret
          name: xxx-tls
    allowedRoutes:
      namespaces:
        from: All
EOF

kubectl apply -f gateway-public.yaml

# 部署 echo ，打印请求信息
cat > 02-deploy-test.yaml <<\EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: echo
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: echo
  template:
    metadata:
      labels:
        app: echo
    spec:
      containers:
      - name: echo
        image: ealen/echo-server:latest
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: echo-svc
  namespace: default
spec:
  selector:
    app: echo
  ports:
  - port: 80
    targetPort: 80
    name: http
  type: ClusterIP
EOF

kubectl  apply -f 02-deploy-test.yaml

# 创建 gateway 规则
cat <<\EOF>> 03-httproute.yaml
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: ngfapp-httproute
  namespace: default
spec:
  parentRefs:
  - name: nginx-gateway-test
    namespace: default
  hostnames:
  - "echo.jasper.org"
  rules:
  - backendRefs:
    - name: echo-svc
      port: 80
      kind: Service
    matches:
    - path:
        type: PathPrefix
        value: /
EOF

root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  apply -f 03-httproute.yaml


root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  get httproute
NAME         HOSTNAMES          AGE
echo-route   ["www.echo.com"]   11m


# 排查。查看 gateway 对应的 pod 日志。
kubectl logs -f nginx-gateway-test-nginx-79d64d4cc4-kcp5q

# 查看 nginx 配置文件
root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  exec -it nginx-gateway-test-nginx-79d64d4cc4-kcp5q -c nginx -- nginx -T
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# configuration file /etc/nginx/nginx.conf:
load_module modules/ngx_http_js_module.so;
include /etc/nginx/main-includes/*.conf;

worker_processes auto;

pid /var/run/nginx/nginx.pid;

# 真实客户端 ip
    "headers": {
      "host": "www.echo.com",
      "x-forwarded-for": "1.1.162.204",
      "x-real-ip": "1.1.162.204",


# 或在 pod echo 所在主机使用 tcpdump 命令查看
tcpdump -i any -nn -vvv -A dst host 172.31.82.72 and port 80  |grep -C50 echo

#nginx-gateway-fabric 2.6.7

cat >values-ngf.yaml <<\EOF
nginxGateway:
  snippets:
    enable: true

nginx:
  service:
    type: LoadBalancer
    externalTrafficPolicy: Cluster
  patches:
  - type: JSONPatch
    value:
    - op: add
      path: /spec/template/spec/volumes/-
      value:
        name: tz-shanghai
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
          type: File
    - op: add
      path: /spec/template/spec/containers/0/volumeMounts/-
      value:
        name: tz-shanghai
        mountPath: /etc/localtime
        readOnly: true

  config:
    logging:
      accessLog:
        escape: json
        format: >-
          {"time":"$time_iso8601","remote_addr":"$remote_addr","x_forwarded_for":"$http_x_forwarded_for","host":"$host","method":"$request_method","uri":"$request_uri","protocol":"$server_protocol","status":$status,"body_bytes_sent":$body_bytes_sent,"request_length":$request_length,"request_time":$request_time,"upstream_addr":"$upstream_addr","upstream_status":"$upstream_status","upstream_response_time":"$upstream_response_time","referer":"$http_referer","user_agent":"$http_user_agent","request_id":"$request_id"}
EOF

# 升级：
helm upgrade --install ngf . \
  --create-namespace -n nginx-gateway \
  -f values-ngf.yaml \
  --wait --timeout 5m

# 配置说明
#nginxGateway.snippets.enable 同时启用 SnippetsFilter 和 SnippetsPolicy。
#nginxGateway.snippetsFilters.enable 已被 chart 标记为 deprecated，且只开前者
#nginx.service.type=LoadBalancer 使用负载均衡器
#nginx.service.externalTrafficPolicy=Cluster 外部流量转发策略
#  Cluster（这里配置的）：源 IP 会被 SNAT 改写为节点 IP；可以做负载均衡跨节点转发。
#  Local：保留真实客户端源 IP，但流量只能落到后端 Pod 所在节点。

# ---- 时区：东八区 ----
# 数据面镜像是 Alpine，既没有 /etc/localtime 也没有 tzdata，
# 所以 $time_iso8601 / $time_local / error log 的时间全是 UTC
# （日志里是 +00:00，比北京时间晚 8 小时）。
#
# 只设 TZ=Asia/Shanghai 是没用的：musl 会去找
# /usr/share/zoneinfo/Asia/Shanghai，镜像里没有这个文件，就静默回退 UTC。
# 必须把宿主机的时区文件挂进来。这里刻意不设 TZ 环境变量——
# musl 在 TZ 未设时直接读 /etc/localtime，正好是我们挂进去的那个。
#
# 用 hostPath 挂宿主机的 /usr/share/zoneinfo/Asia/Shanghai 这个实际文件
#
# NginxProxy CRD 的 container 字段只有 debug/hostPorts/image，给不了
# env 和 volumeMounts，所以只能走 patches。
# chart 会把 nginx.patches 渲染进 NginxProxy 的 kubernetes.deployment.patches。
# 必须用 JSONPatch，不能用 StrategicMerge（默认类型）。
# StrategicMerge 在 NGF provisioner 这条路径上会**按索引覆盖**而不是按 name 合并：
# 只含一个元素的 volumes 会盖掉 volumes[0]（原本是 projected 类型的 token），
# API Server 报 "may not specify more than 1 volume type"，
# 同时 volumeMounts 的引用全部错位，Deployment 更新直接被拒。
# JSONPatch 的 "/-" 是追加到列表末尾，不碰任何现有元素。
#
# containers/0 就是 nginx：数据面 Pod 只有这一个业务容器（另有 init 初始化容器）。

# ---- 日志 json 格式化 ----
# nginx.config 的内容会被原样渲染进 NginxProxy 的 spec
# （chart 模板 templates/nginxproxy.yaml: toYaml .Values.nginx.config）
# nginx.config.logging.accessLog.escape=json —— 让 nginx 按 JSON 规则转义变量值（引号、反斜杠、控制字符），
# 否则 User-Agent 里带引号就会把整行 JSON 破坏掉。
# 注意两条硬性限制（来自 NginxProxy CRD 的 schema）：
#   1. 不能有单引号 —— 这段会被渲染进单引号包裹的 log_format 指令里
#   2. 不能有换行 —— 必须写成一行
#
# 字段取舍：$status / $body_bytes_sent / $request_length / $request_time
# 这几个 nginx 保证非空，所以不加引号，让它们在 JSON 里是数字类型，
# Grafana 里可直接做数值比较和聚合。
# upstream 相关变量在未走到上游时会是空值，必须加引号，否则
# 会产生 "upstream_status":, 这种非法 JSON。

# 都重写为 $2 路径
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
  name: gateway-ingress
  namespace: default
spec:
  ingressClassName: nginx
  rules:
  - host: devg.xxx.me
    http:
      paths:
      - backend:
          service:
            name: cloud-gateway
            port:
              number: 8080
        path: /(api)/(.*)
        pathType: ImplementationSpecific
      - backend:
          service:
            name: wallet-admin-server
            port:
              number: 8080
        path: /(api)/alpha/v1/(system/exchangeRateFacede/.*)
        pathType: ImplementationSpecific

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: gateway-httproute
  namespace: default
spec:
  hostnames:
  - devg.xxx.me
  parentRefs:
  - group: gateway.networking.k8s.io
    kind: Gateway
    name: nginx-gateway-test
    namespace: default
  rules:
  - backendRefs:
    - group: ""
      kind: Service
      name: cloud-gateway
      port: 8080
      weight: 1
    filters:
    - type: URLRewrite
      urlRewrite:
        path:
          replacePrefixMatch: /
          type: ReplacePrefixMatch
    matches:
    - path:
        type: PathPrefix
        value: /api/
  - backendRefs:
    - group: ""
      kind: Service
      name: wallet-admin-server
      port: 8080
      weight: 1
    filters:
    - type: URLRewrite
      urlRewrite:
        path:
          replacePrefixMatch: /system/exchangeRateFacede/
          type: ReplacePrefixMatch
    matches:
    - path:
        type: PathPrefix
        value: /api/alpha/v1/system/exchangeRateFacede/

cat h5-web-antd-httproute.yaml 
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: h5-web-antd-httproute
  namespace: default
spec:
  hostnames:
  - xxx.com
  - devpt.xxx.com
  parentRefs:
  - name: nginx-gateway-test
    namespace: default
  rules:
  - backendRefs:
    - name: h5-web-antd-svc
      port: 80
    matches:
    - path:
        type: PathPrefix
        value: /
---
apiVersion: gateway.nginx.org/v1alpha1
kind: ClientSettingsPolicy
metadata:
  name: h5-web-antd-client-policy
  namespace: default
spec:
  targetRef:
    group: gateway.networking.k8s.io
    kind: HTTPRoute
    name: h5-web-antd-httproute
  body:
    maxSize: "100m"


kubectl  get clientsettingspolicies.gateway.nginx.org h5-web-antd-client-policy

# 查看 nginx 对应配置
kubectl  exec  nginx-gateway-test-nginx-5f5dcf8454-9tdlp -c nginx -- nginx -T 
server {
    listen 80 proxy_protocol;
    listen [::]:80 proxy_protocol;

    server_name devpt.xxx.com;
    set_real_ip_from 172.31.0.0/16;
    real_ip_header proxy_protocol;

        
    location / {
        

        
        include /etc/nginx/includes/ClientSettingsPolicy_default_h5-web-antd-client-policy.conf;

        
# configuration file /etc/nginx/includes/ClientSettingsPolicy_default_h5-web-antd-client-policy.conf:

client_max_body_size 100m;

gzip on;

# 4 是性价比拐点：再往上 CPU 涨得快、体积只多省几个百分点
gzip_comp_level 4;

# 小响应压了反而更大（gzip 头部开销约 20 字节）
gzip_min_length 1k;

# 必须显式开。默认 off 时，带 Via 头的代理响应一律不压——
# 而网关本身就是反向代理，不加这行等于没开
gzip_proxied any;

# 让 CDN / 中间代理按 Accept-Encoding 分别缓存，
# 避免把压缩版发给不支持的客户端
gzip_vary on;

# text/html 不用列，nginx 总是压缩它。
# 刻意【不包含】已经是压缩格式的类型：
#   图片 png/jpg/gif/webp、字体 woff2、视频、zip/gz
# 对它们再压一遍只烧 CPU，体积还可能变大。
gzip_types
  text/plain
  text/css
  text/xml
  text/javascript
  application/json
  application/javascript
  application/xml
  application/rss+xml
  application/atom+xml
  application/manifest+json
  application/vnd.api+json
  image/svg+xml;