Docker集群搭建以及踩坑记录——门店掌柜项目环境

发表于 分类于 阅读次数： 本文字数： 10k 阅读时长 ≈ 9 分钟

前言

门店掌柜项目的编写暂时告一段落，接下来就是把项目搭建起来，这里首先把项目依赖的环境搭建集群，但遇到的坑真实令人泪目···

踩坑汇总

1、Docker搭建集群一定要局域网内的机器，否则集群网络无法互通

2、使用Docker-compose创建集群死活用不了NFS数据卷

注意：集群模式下本地数据卷没用，必须使用文件共享系统

原因：Docker与Docker-compose版本不对应的问题

版本对应表：

Docker-compose版本Docker版本
3.819.03.0+
3.718.06.0+
3.618.02.0+
3.517.12.0+
3.417.09.0+
3.317.06.0+
3.217.04.0+
3.11.13.1+
3.01.13.0+
2.417.12.0+
2.317.06.0+
2.21.13.0+
2.11.12.0+
2.01.10.0+

参考：

https://docs.docker.com/compose/compose-file/compose-versioning/

解决方式：使用对应版本即可

安装指定版本Docker：

1
2
3
4
5
6
7
8
9
10
11
12
# 关闭Docker服务
systemctl stop docker.service

# 安装指定版本
yum downgrade --setopt=obsoletes=0 -y docker-ce-19.03.15-3.el7 docker-ce-cli-19.03.15-3.el7 containerd.io-1.3.9-3.1.el7

# 启动服务
systemctl start docker.service

# 查看版本
docker -v

参考：https://blog.csdn.net/jiongshang3743/article/details/118998571

开始搭建环境

1、搭建NFS文件共享服务

参考：docker高级之——Docker-Compose、Docker-Swarm、Jakins持续集成、Rancher持续部署

  • vim /etc/exports配置如下
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/mnt/nfs/nginx/conf *(rw,no_root_squash,no_all_squash,sync)
/mnt/nfs/nginx/home *(rw,no_root_squash,no_all_squash,sync)

/mnt/nfs/mysql/mydir *(rw,no_root_squash,no_all_squash,sync)
/mnt/nfs/mysql/data *(rw,no_root_squash,no_all_squash,sync)
/mnt/nfs/mysql/conf *(rw,no_root_squash,no_all_squash,sync)
/mnt/nfs/mysql/source *(rw,no_root_squash,no_all_squash,sync)

/mnt/nfs/seata/config *(rw,no_root_squash,no_all_squash,sync)

/mnt/nfs/rabbitmq/data *(rw,no_root_squash,no_all_squash,sync)

/mnt/nfs/xxlJob/data *(rw,no_root_squash,no_all_squash,sync)

/mnt/nfs/redis/data *(rw,no_root_squash,no_all_squash,sync)

/mnt/nfs/waitFor *(rw,no_root_squash,no_all_squash,sync)

注意：需要同时创建相应目录

2、docker-compose.yml文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
version: '3.8'
services:
  #mysql数据库脚本（首次搭建因为要导入sql会导致nacos运行了，但数据库还没创建好，所以首次创建集群得删除重新创建一次）
  mysql:
    image: mysql:5.7
    restart: always
    ports:
      - 3306:3306
    volumes:
      - mysql-nfs-mydir:/mydir
      - mysql-nfs-data:/var/lib/mysql
      - mysql-nfs-conf:/etc
      - mysql-nfs-source:/docker-entrypoint-initdb.d
    environment:
      MYSQL_ROOT_PASSWORD: 123
    networks:
      shopkeeper:
        ipv4_address: 172.21.0.2
    deploy:
       replicas: 1
  #nacos服务脚本
  nacos:
    image: nacos/nacos-server:1.4.0
    restart: always
    ports:
      - "8848:8848"
    volumes:
      - waitFor-nfs:/home/wait-for
    environment:
      SPRING_DATASOURCE_PLATFORM: mysql #数据源平台 仅支持mysql或不保存empty
      MODE: standalone
      MYSQL_SERVICE_HOST: mysql
      MYSQL_SERVICE_DB_NAME: nacos
      MYSQL_SERVICE_PORT: 3306
      MYSQL_SERVICE_USER: root
      MYSQL_SERVICE_PASSWORD: 123
      NACOS_APPLICATION_PORT: 8848
      JVM_XMS: 512m
      JVM_MMS: 256m
      JVM_XMN: 128m
    networks:
      shopkeeper:
        ipv4_address: 172.21.0.3
    deploy:
       replicas: 1
    depends_on:
      - mysql
    # 执行wait-for脚本，检查mysql是否启动成功，成功后执行Nacos启动脚本
    command: sh -c '/home/wait-for/wait-for.sh mysql:3306 -- ./bin/docker-startup.sh'
  xxl-job:
    image: xuxueli/xxl-job-admin:2.1.2
    restart: always
    ports:
      - 8280:8080
    volumes:
      - xxlJob-nfs-data:/data/applogs
      - waitFor-nfs:/home/wait-for
    environment:
      PARAMS: "--spring.datasource.url=jdbc:mysql://mysql:3306/xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai --spring.datasource.username=root --spring.datasource.password=123"
    networks:
      shopkeeper:
        ipv4_address: 172.21.0.4
    depends_on:
      - mysql
    # 执行wait-for脚本，检查mysql是否启动成功，成功后执行Nacos启动脚本
    command: sh -c '/home/wait-for/wait-for.sh mysql:3306 -- ./bin/docker-startup.sh'
  #seata服务脚本
  seata:
    image: seataio/seata-server:1.3.0
    restart: always
    ports:
      - "9200:9200"
    volumes:
      - seata-nfs-config:/root/seata-config
      - waitFor-nfs:/home/wait-for
    environment:
      SEATA_IP: 192.168.200.131
      SEATA_CONFIG_NAME: file:/root/seata-config/registry
      SEATA_PORT: 9200
    networks:
      shopkeeper:
        ipv4_address: 172.21.0.5
    deploy:
       replicas: 1
    depends_on:
      - nacos
    # 执行wait-for脚本，检查mysql是否启动成功，成功后执行Nacos启动脚本
    command: sh -c '/home/wait-for/wait-for.sh nacos:8848 -- ./bin/docker-startup.sh'
  #  rabbitmq脚本
  rabbitmq:
    image: rabbitmq:3.8.3-management
    restart: always
    ports:
      - 15672:15672
      - 5672:5672
    volumes:
      - rabbitmq-nfs-data:/var/lib/rabbitmq
    environment:
      RABBITMQ_DEFAULT_USER: admin
      RABBITMQ_DEFAULT_PASS: 2428933105
    networks:
      shopkeeper:
        ipv4_address: 172.21.0.6
    deploy:
       replicas: 1
  redis:
    image: redis:5.0.0
    restart: always
    command: redis-server --requirepass pass
    ports:
      - 6379:6379
    volumes:
      - redis-nfs-data:/data
    networks:
      shopkeeper:
        ipv4_address: 172.21.0.7
    deploy:
       replicas: 3
  nginx:
    image: nginx:1.18.0
    restart: always
    ports:
      - 80:80
    privileged: true
    volumes:
      - nginx-nfs-conf:/etc/nginx
      - nginx-nfs-home:/home
    networks:
      shopkeeper:
        ipv4_address: 172.21.0.8
    deploy:
       replicas: 1

volumes:
  mysql-nfs-mydir:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/mysql/mydir"
  mysql-nfs-data:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/mysql/data"
  mysql-nfs-conf:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/mysql/conf"
  mysql-nfs-source:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/mysql/source"
  xxlJob-nfs-data:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/xxlJob/data"
  seata-nfs-config:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/seata/config"
  rabbitmq-nfs-data:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/rabbitmq/data"
  redis-nfs-data:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/redis/data"
  nginx-nfs-conf:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/nginx/conf"
  nginx-nfs-home:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/nginx/home"
  waitFor-nfs:
    driver: local
    driver_opts:
      type: nfs
      o: nfsvers=4,addr=192.168.200.131
      device: ":/mnt/nfs/waitFor"
#docker容器内网地址
networks:
  shopkeeper:
    name: shopkeeper
    ipam:
      config:
        - subnet: 172.21.0.0/16

3、wait-for工具

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#!/bin/sh

# The MIT License (MIT)
#
# Copyright (c) 2017 Eficode Oy
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

VERSION="2.2.2"

set -- "$@" -- "$TIMEOUT" "$QUIET" "$PROTOCOL" "$HOST" "$PORT" "$result"
# 设置超时时间，可以设置高一些，单位：秒
TIMEOUT=360
QUIET=0
# The protocol to make the request with, either "tcp" or "http"
PROTOCOL="tcp"

echoerr() {
  if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi
}

usage() {
  exitcode="$1"
  cat << USAGE >&2
Usage:
  $0 host:port|url [-t timeout] [-- command args]
  -q | --quiet                        Do not output any status messages
  -t TIMEOUT | --timeout=timeout      Timeout in seconds, zero for no timeout
  -v | --version                      Show the version of this tool
  -- COMMAND ARGS                     Execute command with args after the test finishes
USAGE
  exit "$exitcode"
}

wait_for() {
  case "$PROTOCOL" in
    tcp)
      if ! command -v nc >/dev/null; then
        echoerr 'nc command is missing!'
        exit 1
      fi
      ;;
    wget)
      if ! command -v wget >/dev/null; then
        echoerr 'wget command is missing!'
        exit 1
      fi
      ;;
  esac

  TIMEOUT_END=$(($(date +%s) + TIMEOUT))

  while :; do
    case "$PROTOCOL" in
      tcp)
        nc -w 1 -z "$HOST" "$PORT" > /dev/null 2>&1
        ;;
      http)
        wget --timeout=1 -q "$HOST" -O /dev/null > /dev/null 2>&1
        ;;
      *)
        echoerr "Unknown protocol '$PROTOCOL'"
        exit 1
        ;;
    esac

    result=$?

    if [ $result -eq 0 ] ; then
      if [ $# -gt 7 ] ; then
        for result in $(seq $(($# - 7))); do
          result=$1
          shift
          set -- "$@" "$result"
        done

        TIMEOUT=$2 QUIET=$3 PROTOCOL=$4 HOST=$5 PORT=$6 result=$7
        shift 7
        exec "$@"
      fi
      exit 0
    fi

    if [ $TIMEOUT -ne 0 -a $(date +%s) -ge $TIMEOUT_END ]; then
      echo "Operation timed out" >&2
      exit 1
    fi

    sleep 1
  done
}

while :; do
  case "$1" in
    http://*|https://*)
    HOST="$1"
    PROTOCOL="http"
    shift 1
    ;;
    *:* )
    HOST=$(printf "%s\n" "$1"| cut -d : -f 1)
    PORT=$(printf "%s\n" "$1"| cut -d : -f 2)
    shift 1
    ;;
    -v | --version)
    echo $VERSION
    exit
    ;;
    -q | --quiet)
    QUIET=1
    shift 1
    ;;
    -q-*)
    QUIET=0
    echoerr "Unknown option: $1"
    usage 1
    ;;
    -q*)
    QUIET=1
    result=$1
    shift 1
    set -- -"${result#-q}" "$@"
    ;;
    -t | --timeout)
    TIMEOUT="$2"
    shift 2
    ;;
    -t*)
    TIMEOUT="${1#-t}"
    shift 1
    ;;
    --timeout=*)
    TIMEOUT="${1#*=}"
    shift 1
    ;;
    --)
    shift
    break
    ;;
    --help)
    usage 0
    ;;
    -*)
    QUIET=0
    echoerr "Unknown option: $1"
    usage 1
    ;;
    *)
    QUIET=0
    echoerr "Unknown argument: $1"
    usage 1
    ;;
  esac
done

if ! [ "$TIMEOUT" -ge 0 ] 2>/dev/null; then
  echoerr "Error: invalid timeout '$TIMEOUT'"
  usage 3
fi

case "$PROTOCOL" in
  tcp)
    if [ "$HOST" = "" ] || [ "$PORT" = "" ]; then
      echoerr "Error: you need to provide a host and port to test."
      usage 2
    fi
  ;;
  http)
    if [ "$HOST" = "" ]; then
      echoerr "Error: you need to provide a host to test."
      usage 2
    fi
  ;;
esac

wait_for "$@"