kubernetes环境中部署rocketmq多主多从集群

rocketmq的多主多从的优势这里就不多说了，在kubernetes中部署rocketmq的有点就是负载自动启动。这里使用两主两从，如果需要增加只需要负载一份主从即可，看一下yaml文件内容。

 [root@k8s-master conf]# cat broker-a.yaml
 ---
 apiVersion: v1
 data:
   broker.conf: |-
     brokerClusterName=DefaultCluster
     brokerName=broker-a
     brokerId=0
     namesrvAddr=mq-ns.qa-asfp.svc.cluster.local:9876
     defaultTopicQueueNums=4
     #Whether to allow the broker to automatically create topics, it is recommended to open offline and close online
     autoCreateTopicEnable=true
     #Whether to allow the broker to automatically create a subscription group, it is recommended to start offline and close online
     autoCreateSubscriptionGroup=true
     listenPort=10911
     deleteWhen=04
     fileReservedTime=120
     mapedFileSizeCommitLog=1073741824
     mapedFileSizeConsumeQueue=300000
     destroyMapedFileInterval=120000
     redeleteHangedFileInterval=120000
     diskMaxUsedSpaceRatio=88
     maxMessageSize=65536
     flushCommitLogLeastPages=4
     flushConsumeQueueLeastPages=2
     flushCommitLogThoroughInterval=10000
     flushConsumeQueueThoroughInterval=60000
     checkTransactionMessageEnable=false
     sendMessageThreadPoolNums=16
     pullMessageThreadPoolNums=16
     brokerRole=ASYNC_MASTER
     flushDiskType=ASYNC_FLUSH
 kind: ConfigMap
 metadata:
   creationTimestamp: null
   name: broker-a
 [root@k8s-master conf]# cat broker-a-s.yaml
 ---
 apiVersion: v1
 data:
   broker.conf: |-
     brokerClusterName=DefaultCluster
     brokerName=broker-a
     brokerId=1
     namesrvAddr=mq-ns.qa-asfp.svc.cluster.local:9876
     defaultTopicQueueNums=4
     autoCreateTopicEnable=true
     autoCreateSubscriptionGroup=true
     listenPort=10911
     deleteWhen=04
     fileReservedTime=120
     mapedFileSizeCommitLog=1073741824
     mapedFileSizeConsumeQueue=300000
     destroyMapedFileInterval=120000
     redeleteHangedFileInterval=120000
     diskMaxUsedSpaceRatio=88
     maxMessageSize=65536
     flushCommitLogLeastPages=4
     flushConsumeQueueLeastPages=2
     flushCommitLogThoroughInterval=10000
     flushConsumeQueueThoroughInterval=60000
     checkTransactionMessageEnable=false
     sendMessageThreadPoolNums=16
     pullMessageThreadPoolNums=16
     brokerRole=SLAVE
     flushDiskType=ASYNC_FLUSH
 kind: ConfigMap
 metadata:
   creationTimestamp: null
   name: broker-a-s
 [root@k8s-master conf]# cat broker-b.yaml
 ---
 apiVersion: v1
 data:
   broker.conf: |-
     brokerClusterName=DefaultCluster
     brokerName=broker-b
     brokerId=0
     namesrvAddr=mq-ns.qa-asfp.svc.cluster.local:9876
     defaultTopicQueueNums=4
     autoCreateTopicEnable=true
     autoCreateSubscriptionGroup=true
     listenPort=10911
     deleteWhen=04
     fileReservedTime=120
     mapedFileSizeCommitLog=1073741824
     mapedFileSizeConsumeQueue=300000
     destroyMapedFileInterval=120000
     redeleteHangedFileInterval=120000
     diskMaxUsedSpaceRatio=88
     maxMessageSize=65536
     flushCommitLogLeastPages=4
     flushConsumeQueueLeastPages=2
     flushCommitLogThoroughInterval=10000
     flushConsumeQueueThoroughInterval=60000
     checkTransactionMessageEnable=false
     sendMessageThreadPoolNums=16
     pullMessageThreadPoolNums=16
     brokerRole=ASYNC_MASTER
     flushDiskType=ASYNC_FLUSH
 kind: ConfigMap
 metadata:
   creationTimestamp: null
   name: broker-b
 [root@k8s-master conf]# cat broker-b-s.yaml
 ---
 apiVersion: v1
 data:
   broker.conf: |-
     brokerClusterName=DefaultCluster
     brokerName=broker-b
     brokerId=1
     namesrvAddr=mq-ns.qa-asfp.svc.cluster.local:9876
     defaultTopicQueueNums=4
     autoCreateTopicEnable=true
     autoCreateSubscriptionGroup=true
     listenPort=10911
     deleteWhen=04
     fileReservedTime=120
     mapedFileSizeCommitLog=1073741824
     mapedFileSizeConsumeQueue=300000
     destroyMapedFileInterval=120000
     redeleteHangedFileInterval=120000
     diskMaxUsedSpaceRatio=88
     maxMessageSize=65536
     flushCommitLogLeastPages=4
     flushConsumeQueueLeastPages=2
     flushCommitLogThoroughInterval=10000
     flushConsumeQueueThoroughInterval=60000
     checkTransactionMessageEnable=false
     sendMessageThreadPoolNums=16
     pullMessageThreadPoolNums=16
     brokerRole=SLAVE
     flushDiskType=ASYNC_FLUSH
 kind: ConfigMap
 metadata:
   creationTimestamp: null
   name: broker-b-s

这是两个主的配置文件，下面是pod的yaml文件

 [root@k8s-master pod]# cat broker-a-pods.yaml
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     app: broker-a
   name: broker-a
 spec:
   # type: NodePort
   type: ClusterIP
   ports:
   - port: 10911
     targetPort: 10911
     name: broker-port
   - port: 10909
     targetPort: 10909
     name: broker-vip
   - port: 10912
     targetPort: 10912
     name: broker-hp
   selector:
     app: broker-a
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: broker-a
 spec:
   serviceName: broker-a
   replicas: 1
   selector:
     matchLabels:
       app: broker-a
   template:
     metadata:
      labels:
        app: broker-a
     spec:
       # affinity:
       #   #podAntiAffinity 反亲合性
       #   podAntiAffinity:
       #     requiredDuringSchedulingIgnoredDuringExecution:
       #       - labelSelector:
       #           matchExpressions:
       #             - key: "app"
       #               operator: In
       #               values:
       #                 - broker-a
       #         topologyKey: "kubernetes.io/hostname"
       containers:
       - name: broker-a
         image: apache/rocketmq:4.9.7
         imagePullPolicy: IfNotPresent
         env:
           - name: TZ
             value: Asia/Shanghai
           - name: NAMESRV_ADDR
             value: "mq-ns.ops.svc.cluster.local:9876"
           - name: JAVA_OPT_EXT
             value: "-Duser.home=/home/rocketmq -Xms512M -Xmx512M -Xmn128m"
         #command: ["sh","-c","mqbroker -c /home/broker.conf"]
         command:
           - /bin/sh
         args:
           - mqbroker
           - '-c'
           - /home/broker.conf
         volumeMounts:
           - name: rocketmq-data
             mountPath: /home/rocketmq/logs
           - name: rocketmq-data
             mountPath: /home/rocketmq/store
           - name: broker-config
             mountPath: /home/broker.conf
             subPath: broker.conf
         readinessProbe:
           tcpSocket:
             port: 10911
           initialDelaySeconds: 15
           timeoutSeconds: 5
           periodSeconds: 20
       volumes:
       - name: broker-config
         configMap:
           name: broker-a
       #启动依赖
       initContainers:
       - name: init-mq-ns
         image: busybox:latest
         command: ['sh', '-c', "until nslookup mq-ns.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for mq-ns; sleep 2; done"]
 ​
   volumeClaimTemplates:
   - metadata:
       name: rocketmq-data
       annotations:
         volume.beta.kubernetes.io/storage-class: "nfs-storage"
     spec:
       accessModes:
         - ReadWriteMany
       resources:
         requests:
           storage: 10Gi
 [root@k8s-master pod]# cat broker-a-s-pods.yaml
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     app: broker-a-s
   name: broker-a-s
 spec:
   # type: NodePort
   type: ClusterIP
   ports:
   - port: 10911
     targetPort: 10911
     name: broker-port
   - port: 10909
     targetPort: 10909
     name: broker-vip
   - port: 10912
     targetPort: 10912
     name: broker-hp
   selector:
     app: broker-a-s
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: broker-a-s
 spec:
   serviceName: broker-a-s
   replicas: 1
   selector:
     matchLabels:
       app: broker-a-s
   template:
     metadata:
      labels:
        app: broker-a-s
     spec:
       # affinity:
       #   #podAntiAffinity 反亲合性
       #   podAntiAffinity:
       #     requiredDuringSchedulingIgnoredDuringExecution:
       #       - labelSelector:
       #           matchExpressions:
       #             - key: "app"
       #               operator: In
       #               values:
       #                 - broker-a-s
       #         topologyKey: "kubernetes.io/hostname"
       containers:
       - name: broker-a-s
         image: apache/rocketmq:4.9.7   #下面使用的是用官方地址自行构建的镜像，推荐直接使用这个
         #image: apacherocketmq/rocketmq:4.8.0
         imagePullPolicy: IfNotPresent
         env:
           - name: TZ
             value: Asia/Shanghai
           - name: NAMESRV_ADDR
             value: "mq-ns.ops.svc.cluster.local:9876"
           - name: JAVA_OPT_EXT
             value: "-Duser.home=/home/rocketmq -Xms512M -Xmx512M -Xmn128m"
         #command: ["sh","-c","mqbroker -c /home/broker.conf"]
         command:
           - /bin/sh
         args:
           - mqbroker
           - '-c'
           - /home/broker.conf
         volumeMounts:
           - name: rocketmq-data
             mountPath: /home/rocketmq/logs
           - name: rocketmq-data
             mountPath: /home/rocketmq/store
           - name: broker-config
             mountPath: /home/broker.conf
             subPath: broker.conf
         readinessProbe:
           tcpSocket:
             port: 10911
           initialDelaySeconds: 15
           timeoutSeconds: 5
           periodSeconds: 20
       volumes:
       - name: broker-config
         configMap:
           name: broker-a-s
         #启动依赖
       initContainers:
       - name: init-mq-ns
         image: busybox:latest
         command: ['sh', '-c', "until nslookup mq-ns.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for mq-ns; sleep 2; done"]
 ​
   volumeClaimTemplates:
   - metadata:
       name: rocketmq-data
       annotations:
         volume.beta.kubernetes.io/storage-class: "nfs-storage"
     spec:
       accessModes:
         - ReadWriteMany
       resources:
         requests:
           storage: 10Gi
 [root@k8s-master pod]# cat broker-b-pods.yaml
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     app: broker-b
   name: broker-b
 spec:
   # type: NodePort
   type: ClusterIP
   ports:
   - port: 10911
     targetPort: 10911
     name: broker-port
   - port: 10909
     targetPort: 10909
     name: broker-vip
   - port: 10912
     targetPort: 10912
     name: broker-hp
   selector:
     app: broker-b
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: broker-b
 spec:
   serviceName: broker-b
   replicas: 1
   selector:
     matchLabels:
       app: broker-b
   template:
     metadata:
      labels:
        app: broker-b
     spec:
       # affinity:
       #   #podAntiAffinity 反亲合性
       #   podAntiAffinity:
       #     requiredDuringSchedulingIgnoredDuringExecution:
       #       - labelSelector:
       #           matchExpressions:
       #             - key: "app"
       #               operator: In
       #               values:
       #                 - broker-b
       #         topologyKey: "kubernetes.io/hostname"
       containers:
       - name: broker-b
         image: apache/rocketmq:4.9.7   #下面使用的是用官方地址自行构建的镜像，推荐直接使用这个
         #image: apacherocketmq/rocketmq:4.8.0
         imagePullPolicy: IfNotPresent
         env:
           - name: TZ
             value: Asia/Shanghai
           - name: NAMESRV_ADDR
             value: "mq-ns.ops.svc.cluster.local:9876"
           - name: JAVA_OPT_EXT
             value: "-Duser.home=/home/rocketmq -Xms512M -Xmx512M -Xmn128m"
         #command: ["sh","-c","mqbroker -c /home/broker.conf"]
         command:
           - /bin/sh
         args:
           - mqbroker
           - '-c'
           - /home/broker.conf
         volumeMounts:
           - name: rocketmq-data
             mountPath: /home/rocketmq/logs
           - name: rocketmq-data
             mountPath: /home/rocketmq/store
           - name: broker-config
             mountPath: /home/broker.conf
             subPath: broker.conf
         readinessProbe:
           tcpSocket:
             port: 10911
           initialDelaySeconds: 15
           timeoutSeconds: 5
           periodSeconds: 20
       volumes:
       - name: broker-config
         configMap:
           name: broker-b
         #启动依赖
       initContainers:
       - name: init-mq-ns
         image: busybox:latest
         command: ['sh', '-c', "until nslookup mq-ns.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for mq-ns; sleep 2; done"]
 ​
   volumeClaimTemplates:
   - metadata:
       name: rocketmq-data
       annotations:
         volume.beta.kubernetes.io/storage-class: "nfs-storage"
     spec:
       accessModes:
         - ReadWriteMany
       resources:
         requests:
           storage: 10Gi
 [root@k8s-master pod]# cat broker-b-s-pods.yaml
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     app: broker-b-s
   name: broker-b-s
 spec:
   # type: NodePort
   type: ClusterIP
   ports:
   - port: 10911
     targetPort: 10911
     name: broker-port
   - port: 10909
     targetPort: 10909
     name: broker-vip
   - port: 10912
     targetPort: 10912
     name: broker-hp
   selector:
     app: broker-b-s
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: broker-b-s
 spec:
   serviceName: broker-b-s
   replicas: 1
   selector:
     matchLabels:
       app: broker-b-s
   template:
     metadata:
      labels:
        app: broker-b-s
     spec:
       # affinity:
       #   #podAntiAffinity 反亲合性
       #   podAntiAffinity:
       #     requiredDuringSchedulingIgnoredDuringExecution:
       #       - labelSelector:
       #           matchExpressions:
       #             - key: "app"
       #               operator: In
       #               values:
       #                 - broker-b-s
       #         topologyKey: "kubernetes.io/hostname"
       containers:
       - name: broker-b-s
         image: apache/rocketmq:4.9.7   #下面使用的是用官方地址自行构建的镜像，推荐直接使用这个
         #image: apacherocketmq/rocketmq:4.8.0
         imagePullPolicy: IfNotPresent
         env:
           - name: TZ
             value: Asia/Shanghai
           - name: NAMESRV_ADDR
             value: "mq-ns.ops.svc.cluster.local:9876"
           - name: JAVA_OPT_EXT
             value: "-Duser.home=/home/rocketmq -Xms512M -Xmx512M -Xmn128m"
         #command: ["sh","-c","mqbroker -c /home/broker.conf"]
         command:
           - /bin/sh
         args:
           - mqbroker
           - '-c'
           - /home/broker.conf
         volumeMounts:
           - name: rocketmq-data
             mountPath: /home/rocketmq/logs
           - name: rocketmq-data
             mountPath: /home/rocketmq/store
           - name: broker-config
             mountPath: /home/broker.conf
             subPath: broker.conf
         readinessProbe:
           tcpSocket:
             port: 10911
           initialDelaySeconds: 15
           timeoutSeconds: 5
           periodSeconds: 20
       volumes:
       - name: broker-config
         configMap:
           name: broker-b-s
         #启动依赖
       initContainers:
       - name: init-mq-ns
         image: busybox:latest
         command: ['sh', '-c', "until nslookup mq-ns.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for mq-ns; sleep 2; done"]
 ​
   volumeClaimTemplates:
   - metadata:
       name: rocketmq-data
       annotations:
         volume.beta.kubernetes.io/storage-class: "nfs-storage"
     spec:
       accessModes:
         - ReadWriteMany
       resources:
         requests:
           storage: 10Gi     
 [root@k8s-master conf]# kubectl apply -f . -n ops

rocketmq的namesr是负责调度broker的，这里注意我使用的namespace是ops，所以在自己环境部署的时候要注意一点自己的namespace相应的修改一下即可。

 [root@k8s-master pod]# cat mq-nmserv.yaml
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     app: mq-namesrv
   name: mq-ns
 spec:
   type: ClusterIP
   ports:
   - port: 9876
     targetPort: 9876
     name: namesrv-port
   selector:
     app: mq-namesrv
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: mq-namesrv
 spec:
   serviceName: mq-namesrv
   replicas: 1
   selector:
     matchLabels:
       app: mq-namesrv
   template:
     metadata:
      labels:
        app: mq-namesrv
     spec:
       # affinity:
       #   #podAntiAffinity 反亲合性
       #   podAntiAffinity:
       #     requiredDuringSchedulingIgnoredDuringExecution:
       #       - labelSelector:
       #           matchExpressions:
       #             - key: "app"
       #               operator: In
       #               values:
       #                 - mq-namesrv
       #         topologyKey: "kubernetes.io/hostname"
       containers:
       - name: mq-namesrv
         image: apache/rocketmq:4.9.7
         imagePullPolicy: IfNotPresent
         env:
           - name: TZ
             value: Asia/Shanghai
           - name: JAVA_OPT_EXT
             value: "-Duser.home=/home/rocketmq -Xms512M -Xmx512M -Xmn128m"
         #command: ["sh","-c","mqnamesrv"]
         command:
           - /bin/sh
         args:
           - mqnamesrv
         readinessProbe:
           tcpSocket:
             port: 9876
           initialDelaySeconds: 15
           timeoutSeconds: 5
           periodSeconds: 20

最后一个是rocketmq的控制台，这里用到了连接namesr，所以也要注意自己的namespace，避免连接失败。

 [root@k8s-master pod]# cat mq-console.yaml
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     app: mq-console
   name: mq-cs
 spec:
   type: ClusterIP
   ports:
   - port: 8080
     targetPort: 8080
     name: console-port
   selector:
     app: mq-console
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: mq-console
 spec:
   replicas: 1
   selector:
     matchLabels:
       app: mq-console
   template:
     metadata:
      labels:
        app: mq-console
     spec:
       # affinity:
       #   #podAntiAffinity 反亲合性
       #   podAntiAffinity:
       #     requiredDuringSchedulingIgnoredDuringExecution:
       #       - labelSelector:
       #           matchExpressions:
       #             - key: "app"
       #               operator: In
       #               values:
       #                 - mq-console
       #         topologyKey: "kubernetes.io/hostname"
       containers:
       - name: mq-console
         image: gguilin/rocketmq-console-ng
         imagePullPolicy: IfNotPresent
         env:
           - name: JAVA_OPTS
             value: "-Drocketmq.namesrv.addr=mq-ns.ops.svc.cluster.local:9876 -Dcom.rocketmq.sendMessageWithVIPChannel=false"
         readinessProbe:
           tcpSocket:
             port: 8080
           initialDelaySeconds: 15
       #启动依赖
       initContainers:
       - name: init-mq-ns
         image: busybox:latest
         command: ['sh', '-c', "until nslookup mq-ns.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for mq-ns; sleep 2; done"]
 [root@k8s-master pod]# kubectl apply -f . -n ops

控制台是无状态的服务，其他都是有状态的服务，所以需要用到了数据持久化，可以参考：https://www.wulaoer.org/?p=2714

部署好后，我们看一下控制台的信息，是否正常。

至此rocketmq集群就搭建好了，看些其他的吧