网站Logo 欢迎来到我的博客
首页
关于

自动化运维项目-kodbox(基于tengine)

zhang
23
2024-06-28

网站集群架构图

ansible分发docker/docker-compose

roles目录

编写tasks文件

- name: 1.删除麒麟系统中的runc文件
  file:
    path: /usr/local/bin/runc
    state: absent
- name: 2.创建docker目录
  file:
    path: /etc/docker/
    state: directory
- name: 3.分发配置文件
  copy:
    src: "{{ item.src }}"
    dest: "{{ item.dest }}"
  loop:
    - { src: docker-27.0.3.tgz,dest: /tmp/ } 
    - { src: docker-compose,dest: /tmp/ } 
    - { src: docker.service,dest: /usr/lib/systemd/system/  }
    - { src: daemon.json,dest: /etc/docker/ }
- name: 4.解压docker安装包
  unarchive:
    src: /tmp/docker-27.0.3.tgz
    dest: /tmp/
    owner: root
    group: root
    mode: '0755'
    remote_src: yes
- name: 5.docker命令和docker-compose移动到/bin/
  copy:
    src: "{{ item }}" 
    dest:  /bin/
    owner: root
    group: root
    mode: '0755'
    remote_src: yes
  loop:
    - /tmp/docker/
    - /tmp/docker-compose
- name: 6.重启配置文件
  systemd:
    daemon_reload: yes
- name: 7.重启服务
  systemd:
    name: docker
    enabled: yes
    state: restarted
  • 检查docker与docker-composer
ansible all -m shell -a 'docker --version'
ansible all -m shell -a 'docker-compose --version'

搭建harbor私有镜像仓库

多阶段构建Tengine镜像

Dockerfile目录结构

#创建构建Tengine目录
mkdir -p /app/tools/tengine/
#创建存放nginx配置文件
mkdir -p /app/tools/tengine/conf
#创建Dockerfile文件
vim Dockerfile

Dockerfile多阶段构建镜像

nginx主配置文件

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    keepalive_timeout  65;

    # 包含站点配置目录
    include /etc/nginx/conf.d/*.conf;
}

Tengine构建

#以ubuntu:20.04为基础镜像构建

#######################
# 1. 编译阶段 - 使用标准路径
#######################
FROM ubuntu:20.04 AS builder

LABEL maintainer="Tengine docker admin <362783633@qq.com>" author="zmx"

# 设置环境变量
ENV Web_User        "nginx"
ENV Web_Server      "tengine"
ENV Web_Version     "3.1.0"
ENV DEBIAN_FRONTEND noninteractive
ENV TZ Asia/Shanghai

# 安装编译依赖
RUN sed -ri 's#archive.ubuntu.com|security.ubuntu.com#mirrors.aliyun.com#g' /etc/apt/sources.list \
    && apt-get update \
    && apt-get install -y wget build-essential libssl-dev \
       libpcre3-dev zlib1g-dev \
    && rm -rf /var/lib/apt/lists/*

# 创建运行时用户
RUN groupadd ${Web_User} \
    && useradd -g ${Web_User} -s /bin/false ${Web_User}

# 下载并编译 Tengine - 使用标准安装路径
RUN wget -P /tmp/ http://tengine.taobao.org/download/${Web_Server}-${Web_Version}.tar.gz \
    && tar -xzf /tmp/${Web_Server}-${Web_Version}.tar.gz -C /tmp/ \
    && cd /tmp/${Web_Server}-${Web_Version} \
    && ./configure \
        --prefix=/usr/share/nginx \
        --sbin-path=/usr/sbin/nginx \
        --modules-path=/usr/lib/nginx/modules \
        --conf-path=/etc/nginx/nginx.conf \
        --error-log-path=/var/log/nginx/error.log \
        --http-log-path=/var/log/nginx/access.log \
        --pid-path=/var/run/nginx.pid \
        --lock-path=/var/run/nginx.lock \
        --user=${Web_User} \
        --group=${Web_User} \
        --with-http_ssl_module \
        --with-http_v2_module \
        --with-http_realip_module \
        --with-http_stub_status_module \
        --with-http_mp4_module \
        --with-stream \
        --with-stream_ssl_module \
        --with-stream_realip_module \
        --add-module=modules/ngx_http_upstream_check_module \
        --add-module=modules/ngx_http_upstream_session_sticky_module \
    && make -j $(nproc) \
    && make install \
    && rm -rf /tmp/${Web_Server}-*

#######################
# 2. 最终镜像阶段 - 标准配置路径
#######################
FROM ubuntu:20.04

LABEL maintainer="Tengine docker admin <youjiu_linux@qq.com>" author="xiaozhilidao996"

# 设置环境变量
ENV Web_User        "nginx"
ENV DEBIAN_FRONTEND noninteractive
ENV TZ Asia/Shanghai

# 从编译阶段复制已构建的程序
COPY --from=builder /usr/ /usr/
COPY --from=builder /etc/nginx /etc/nginx
COPY --from=builder /etc/passwd /etc/passwd
COPY --from=builder /etc/group /etc/group

# 添加基础配置文件 (覆盖默认配置)
COPY conf/nginx.conf /etc/nginx/nginx.conf

# 创建标准配置目录结构
RUN mkdir -p /etc/nginx/conf.d \
    && mkdir -p /var/log/nginx \
    && chown -R ${Web_User}:${Web_User} /var/log/nginx \
    # 链接日志到标准输出
    && ln -sf /dev/stdout /var/log/nginx/access.log \
    && ln -sf /dev/stderr /var/log/nginx/error.log \
    # 安装运行时依赖
    && sed -ri 's#archive.ubuntu.com|security.ubuntu.com#mirrors.aliyun.com#g' /etc/apt/sources.list \
    && apt-get update \
    && apt-get install -y --no-install-recommends \
        libpcre3 \
        zlib1g \
        openssl \
        ca-certificates \
    # 清理
    && rm -rf /var/lib/apt/lists/*

# 暴露端口
EXPOSE 80 443

# 启动命令
CMD ["nginx", "-g", "daemon off;"]

上传Tengine镜像到私有镜像仓库

部署kodbox数据库

构建数据库镜像

#创建存放数据库docker-compose.yml文件
mkdir -p /server/docker-compose/kodbox-mysql/

#书写构建数据库镜像
[root@db01 /server/docker-compose/kodbox-mysql]# cat docker-compose.yml 
services:
  kodbox_db:
    image: "mysql:8.0-debian"
    container_name: kodbox_db
    restart: always
    environment:
      MYSQL_ROOT_PASSWORD: "1"
      MYSQL_DATABASE: "kodbox"
      MYSQL_USER: "kodbox"
      MYSQL_PASSWORD: "kodbox"
    ports:
      - 3306:3306  
    volumes:
      - kodbox_db:/var/lib/mysql/
volumes:
    kodbox_db:

#启动镜像
docker-compose up -d 
#查看运行镜像
docker ps -a

上传数据库镜像到私有镜像仓库

#登录到私有镜像仓库
docker login -uadmin -pLidao996 harbor.zmx.cn
#记录tag标签
docker tag mysql:8.0-debian  harbor.zmx.cn/zhangmx/mysql:8.0-debian
#上传镜像
docker push harbor.zmx.cn/zhangmx/mysql:8.0-debian

部署kodbox-redis缓存服务器

启动redis容器

#使用redis:6.0-alpine
[root@redis ~]# docker images | grep redis
redis        6.0-alpine    26c28490dc59   2 years ago   26.6MB
redis        5.0-alpine    7558bc54e8a2   2 years ago   22.9MB
redis        4.0-alpine    e3dd0e49bca5   5 years ago   20.4MB
#启动容器
[root@redis ~]# docker run  -d --name kodbox_redis  -p 6379:6379  redis:6.0-alpine

上传镜像到私有镜像仓库

部署kodbox-web服务器

kodbox目录结构

#wget下载kodbox
mkdir -p /server/docker-compose/kodbox/
#解压kodbox到/server/docker-compose/kodbox/

书写构建kodbox镜像

apt源文件

[root@web01 /server/docker-compose/kodbox]# cat sources.list 
deb http://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy-security main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy-updates main restricted universe multiverse

# deb http://mirrors.aliyun.com/ubuntu/ jammy-proposed main restricted universe multiverse
# deb-src http://mirrors.aliyun.com/ubuntu/ jammy-proposed main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ jammy-backports main restricted universe multiverse

Dockerfile

FROM harbor.zmx.cn/zhangmx/tengine:3.1.0 
LABEL author=zmx  desc="kodbox镜像-tengine-php"


ENV SRC  sources.list
ENV CODE /app/code/
ENV TZ=Asia/Shanghai

ADD ${SRC} /etc/apt/sources.list

RUN    apt update \
    && DEBIAN_FRONTEND=noninteractive  apt install -y tzdata \
    && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
    && echo $TZ > /etc/timezone \
    && mkdir -p  ${CODE}  \
    && apt install -y php8.1-common php8.1-bcmath php8.1-cli php8.1-curl php8.1-dev php8.1-fpm php8.1-gd php8.1-mysql php8.1-mbstring php8.1-redis \
    && ln -sf  /var/log/nginx/access.log /dev/stdout \
    && ln -sf  /var/log/nginx/error.log /dev/stderr  \
    && rm -f /etc/nginx/sites-enabled/default \
    && sed -i 's#^listen =.*#listen = 127.0.0.1:9000#g'  /etc/php/8.1/fpm/pool.d/www.conf \
    && sed -i '/http {/a\\    include /etc/nginx/conf.d/kodbox.conf;' /etc/nginx/nginx.conf
    #删除默认站点  
    #暴漏服务日志

ADD  kodbox.conf  /etc/nginx/conf.d/
ADD  entry.sh  /
ADD  kodbox.tar.gz /app/code/

RUN  chown -R www-data.www-data  /app/code/kodbox/

EXPOSE 80 443 

CMD ["/entry.sh"]

docker-compose

[root@web01 /server/docker-compose/kodbox]# cat docker-compose.yml 
services:
  kodbox_web:
    image: "web:kodbox_v1"
    build:
      context:  .
      dockerfile: Dockerfile
    container_name: kodbox_v1
    ports:
      - 80:80
    restart: always
    #容器持久化
    volumes:
      - data:/app/code/kodbox/data/
volumes:
  data:

kodbox站点目录

[root@web01 /server/docker-compose/kodbox]# cat kodbox.conf 
server {
  listen 80;
  server_name kodbox.zmx.cn;
  root /app/code/kodbox/;
location / {
 index index.php ;
}  
location ~*\.php$ {
 fastcgi_pass 127.0.0.1:9000;
 fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
 include fastcgi_params;
}
}

kodbox服务启动脚本文件

[root@web01 /server/docker-compose/kodbox]# cat entry.sh 
#!/bin/bash
##############################################################
# File Name:entry.sh
# Version:V1.0
# Author:oldboy zmx
# Organization:www.oldboyedu.com
# Desc:
##############################################################
php-fpm8.1
nginx -g "daemon off;"

生成镜像

#构建镜像
docker-compose build
#启动镜像
docker-compose up -d
#查看镜像
docker ps -a


###本机IP访问测试 10.0.0.7:80

上传镜像到私有仓库

#登录私有镜像仓库
docker login -uadmin -pLidao996 harbor.zmx.cn
#镜像标签
docker tag web:kodbox_v1  harbor.zmx.cn/zhangmx/web:kodbox_v1
#上传镜像
docker push harbor.zmx.cn/zhangmx/web:kodbox_v1

本机IP访问测试

IP解析到本地hosts

  • 账号：admin
  • 密码：zmx0000

kodbox接入阿里云OSS存储

接入负载均衡与高可用服务

配置负载均衡

[root@lb01 ~]# cat /etc/nginx/conf.d/kodbox.zmx.cn.conf 
upstream lb_group  {
  server 10.0.0.7:80;
  server 10.0.0.8:80;
}
server {
  listen 80;
  server_name kodbox.zmx.cn;
  error_log /var/log/nginx/kodbox-error.log notice;
  access_log /var/log/nginx/kodbox-access.log main;
  location / {
    proxy_pass http://lb_group;
    proxy_set_header Host $http_host;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Real-Ip $remote_addr;
  }
}

配置高可用服务

[root@lb01 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
#全局定义部分
global_defs {
   router_id lb01
}
#定义监控脚本
vrrp_script keep_lb.sh{
 scripts /server/scripts/keep_lb.sh
 interval 2
 weight 1
 user root
}
#实例名字，主备之间要一致
vrrp_instance vip_3 {
   state MASTER#配置主节点
   interface ens33#指定网卡
   virtual_router_id 51#同一主备之间id要一致
   priority 100#优先级为100，数字越大优先级越高，建议优先级相差50
   advert_int 1
   #简单认证
   authentication {
       auth_type PASS
       auth_pass 1111
   }
   virtual_ipaddress {
        10.0.0.3 dev ens33 label ens33:0
   }
   track_script {
   keep_lb.sh
}
}

高可用VIP解析到本地测试

解决更新kodbox需要重新配置数据库与redis的问题

  • 在构建kodbox容器的时候分发到对应目录下
  • 确保分发前已经书写了该/app/code/变量目录
  • 重新构建镜像，重新启动容器
FROM harbor.zmx.cn/zhangmx/tengine:3.1.0 
LABEL author=zmx  desc="kodbox镜像-tengine-php"


ENV SRC  sources.list
ENV CODE /app/code/
ENV TZ=Asia/Shanghai

ADD ${SRC} /etc/apt/sources.list

RUN    apt update \
    && DEBIAN_FRONTEND=noninteractive  apt install -y tzdata \
    && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
    && echo $TZ > /etc/timezone \
    && mkdir -p  ${CODE}  \
    && apt install -y php8.1-common php8.1-bcmath php8.1-cli php8.1-curl php8.1-dev php8.1-fpm php8.1-gd php8.1-mysql php8.1-mbstring php8.1-redis \
    && ln -sf  /var/log/nginx/access.log /dev/stdout \
    && ln -sf  /var/log/nginx/error.log /dev/stderr  \
    && rm -f /etc/nginx/sites-enabled/default \
    && sed -i 's#^listen =.*#listen = 127.0.0.1:9000#g'  /etc/php/8.1/fpm/pool.d/www.conf \
    && sed -i '/http {/a\\    include /etc/nginx/conf.d/kodbox.conf;' /etc/nginx/nginx.conf
    #删除默认站点  
    #暴漏服务日志

ADD  kodbox.conf  /etc/nginx/conf.d/
ADD  entry.sh  /
ADD  kodbox.tar.gz /app/code/
COPY ./install.lock       /app/code/kodbox/data/system/
COPY ./setting_user.php   /app/code/kodbox/config/

RUN  chown -R www-data.www-data  /app/code/kodbox/

EXPOSE 80 443 

CMD ["/entry.sh"]

创建安全锁文件（只需空白文件）

#安全锁文件/app/code/kodbox/data/system/
touch install.lock

保留连接数据库与redis的setting_user.php文件

#文件在/app/code/kodbox/config/下，第一次配置完数据库与redis会自动生成
<?php
$config['database'] = array (
  'DB_TYPE' => 'mysqli',
  'DB_HOST' => '10.0.0.51',
  'DB_PORT' => 3306,
  'DB_USER' => 'kodbox',
  'DB_PWD' => 'kodbox',
  'DB_NAME' => 'kodbox',
  'DB_SQL_LOG' => true,
  'DB_FIELDS_CACHE' => true,
  'DB_SQL_BUILD_CACHE' => false,
);
$config['cache']['sessionType'] = 'redis';
$config['cache']['cacheType'] = 'redis';
$config['cache']['redis']['host'] = '10.0.0.21';

备份服务器

配置rsync服务

安装rsync服务

yum -y install rsync

rsyncd配置文件

[root@backup ~]# cat /etc/rsyncd.conf
##rsyncd.conf start##
fake super = yes 
uid = www
gid = www
use chroot = no
max connections = 2000
timeout = 600
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /var/log/rsyncd.log
ignore errors
read only = false
list = false
#hosts allow = 10.0.0.0/24
#hosts deny = 0.0.0.0/32
auth users = rsync_backup
secrets file = /etc/rsync.password
#####################################
[kodbox]
comment = path
path = /nfs/backup/kodbox/

创建虚拟用户

密码文件

创建备份目录

测试

NFS存储+实时备份

配置nfs与lsyncd

安装nfs与rpcbind与lsyncd

nfs配置文件

[root@nfs01 ~]# cat /etc/exports
#1.共享/nfs/kodbox/uploads/  目录给172.16.1.0/24网段使用，所有者和用户组为www
/nfs/kodbox/uploads/   172.16.1.0/24(rw,all_squash,anonuid=1999,anongid=1999)

创建虚拟用户

创建存储目录

lsyncd配置文件

[root@nfs01 ~]# cat /etc/lsyncd.conf 
settings {
    logfile = "/var/log/lsyncd.log",
    pidfile = "/var/run/lsyncd.pid",
    statusFile = "/var/log/lsyncd.status",
    nodaemon = true,
    maxProcesses = 2
}
 
sync {
    default.rsync,
    source = "/nfs/kodbox/uploads/",
    target = "rsync_backup@172.16.1.41::kodbox",
    delay = 10,
    delete = true,
    rsync = {
        binary = "/usr/bin/rsync",
        archive = true,
        compress = true,
        password_file = "/etc/rsync.client"
    }
}

kodbox服务器数据挂载

[root@web01 ~]# ll  /var/lib/docker/volumes/kodbox_data/_data/
总用量 0
drwxr-xr-x 4 www www 54  8月 11 10:53 files
-rw-r--r-- 1 www www  0  8月 11 09:35 index.html
-rw-r--r-- 1 www www  0  8月 12 09:52 lidao996.txt
drwxr-xr-x 2 www www 85  8月 11 09:34 system
drwxr-xr-x 7 www www 72  8月 11 17:43 temp
-rw-r--r-- 1 www www  0  8月 15 11:22 xiaozhu996.txt
  • 临时挂载
mount -t nfs nfs01:/nfs/kodbox/uploads/    /var/lib/docker/volumes/kodbox_data/_data/
df -h

  • 永久挂载
#vim /etc/fstab
nfs01:/nfs/kodbox/uploads      /var/lib/docker/volumes/kodbox_data/_data   nfs  defaults  0 0