463 lines
9.0 KiB
Markdown
463 lines
9.0 KiB
Markdown
# Docker Swarm
|
|
|
|
## 1. Compute Modules
|
|
|
|
### Install OS / headless installation
|
|
|
|
1. Flash the image to a SD card/ EMMC.
|
|
2. Go to Boot partition.
|
|
3. Then change the DTB listed in /boot/extlinux/extlinux.conf to -model-a.dtb instead of -cm4.dtb.
|
|
4. Now put the SD card/ EMMC to the preferred base board and boot.
|
|
5. First boot will take time as the user partition will resize to take whole storage size.
|
|
|
|
> ssh as "root" and complete setup
|
|
|
|
```
|
|
sudo apt update && sudo apt upgrade
|
|
|
|
for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done
|
|
|
|
sudo apt install -y docker-compose docker git vim screenfetch net-tools wget curl
|
|
|
|
sudo apt autoremove
|
|
```
|
|
|
|
#### Edit
|
|
|
|
> sudo vim /etc/sudoers.d/pleb
|
|
|
|
```
|
|
pleb ALL=(ALL:ALL) NOPASSWD: ALL
|
|
```
|
|
|
|
### Set Static IP
|
|
|
|
#### Create/Edit
|
|
|
|
> sudo vim /etc/systemd/network/end0.network
|
|
|
|
```
|
|
[Match]
|
|
Name=end0
|
|
|
|
[Network]
|
|
Address=192.168.1.xxx/24
|
|
Gateway=192.168.1.254
|
|
# OpenDNS
|
|
DNS=208.67.222.222
|
|
DNS=208.67.220.220
|
|
```
|
|
|
|
#### Edit
|
|
|
|
> sudo vim /etc/hosts
|
|
|
|
```
|
|
# Host addresses
|
|
127.0.0.1 localhost
|
|
127.0.1.1 sandy
|
|
# Nodes
|
|
192.168.1.14 sandy sandy.local
|
|
192.168.1.13 gary gary.local
|
|
192.168.1.16 sheldon sheldon.local
|
|
192.168.1.19 pearl pearl.local
|
|
```
|
|
|
|
#### Reboot
|
|
|
|
> On remote :
|
|
|
|
```
|
|
ssh-keygen -t ed25519
|
|
ssh-copy-id -f -i ~/.ssh/id_ed25519.pub pleb@<host>
|
|
ssh pleb@<host>
|
|
```
|
|
|
|
#### Edit
|
|
|
|
> sudo vim /etc/ssh/sshd\_config
|
|
|
|
```
|
|
PasswordAuthentication no
|
|
```
|
|
|
|
#### Restart service
|
|
|
|
```
|
|
sudo systemctl restart sshd
|
|
```
|
|
|
|
---
|
|
|
|
```
|
|
sudo systemctl enable iptables
|
|
|
|
sudo systemctl start iptables
|
|
|
|
sudo iptables -I INPUT -p tcp -m tcp --dport 2376 -j ACCEPT
|
|
sudo iptables -I INPUT -p tcp -m tcp --dport 2377 -j ACCEPT
|
|
sudo iptables -I INPUT -p tcp -m tcp --dport 7946 -j ACCEPT
|
|
sudo iptables -I INPUT -p udp -m udp --dport 7946 -j ACCEPT
|
|
sudo iptables -I INPUT -p tcp -m udp --dport 4789 -j ACCEPT
|
|
sudo iptables -I INPUT -p 50 -j ACCEPT # allows ipsec when secure overlay is enabled
|
|
```
|
|
|
|
#### Restart...
|
|
|
|
---
|
|
|
|
## 2. Docker Swarm
|
|
|
|
### Install docker...
|
|
|
|
```bash
|
|
sudo apt install -y docker docker-compose git
|
|
|
|
sudo usermod -aG docker pleb
|
|
|
|
sudo systemctl enable docker.service
|
|
sudo systemctl start docker.service
|
|
|
|
sudo chmod 666 /var/run/docker.sock
|
|
```
|
|
|
|
---
|
|
|
|
#### On master
|
|
|
|
```bash
|
|
docker swarm init --advertise-addr 192.168.1.14
|
|
```
|
|
|
|
> *Swarm initialized: current node (z76p66jcv4c1c9qstodlpcfii) is now a manager.*
|
|
|
|
#### On nodes
|
|
|
|
```bash
|
|
docker swarm join --token SWMTKN-1-27tm53ylrwyesiq3ntfumfixw3gpe2164rnsume8sg07saz12v-1fig1jzehaocodm6hv322l4ku 192.168.1.14:2377
|
|
```
|
|
|
|
> *To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions*.
|
|
|
|
> Change ip range if necessary ?!
|
|
|
|
```bash
|
|
docker network rm ingress
|
|
# Create in different range
|
|
docker network create --driver overlay --ingress --subnet 192.168.1.0/16 --gateway 192.168.1.254 ingress
|
|
```
|
|
|
|
---
|
|
|
|
```bash
|
|
docker node ls
|
|
```
|
|
```
|
|
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
|
|
lqu5nr20g6vt2g1hc5xwm2njg gary Ready Active 29.6.1
|
|
3aqxc8mj1wk2qp1k0llodgyxw pearl Ready Active 29.6.1
|
|
z76p66jcv4c1c9qstodlpcfii * sandy Ready Active Leader 29.6.1
|
|
i62rvg0fh6c0hv49btb1qrfv4 sheldon Ready Active 29.6.1
|
|
```
|
|
|
|
```bash
|
|
docker network ls
|
|
```
|
|
```
|
|
NETWORK ID NAME DRIVER SCOPE
|
|
3b8a522fd1ad bridge bridge local
|
|
8624696f7cbd docker_gwbridge bridge local
|
|
116b95142a3a host host local
|
|
jg83q53aljof ingress overlay swarm
|
|
46f3064ef5dc none null local
|
|
```
|
|
---
|
|
|
|
## 3. Keepalived
|
|
|
|
> *Load Balancer : install on all nodes*
|
|
|
|
```bash
|
|
sudo apt install keepalived -y
|
|
```
|
|
|
|
### On master node (sandy)
|
|
|
|
#### Create/Edit
|
|
|
|
```bash
|
|
sudo vim /etc/keepalived/keepalived.conf
|
|
```
|
|
|
|
```
|
|
! Configuration File for keepalived
|
|
|
|
global_defs {
|
|
notification_email {
|
|
admin@delmar.bzh
|
|
}
|
|
notification_email_from admin@delmar.bzh
|
|
smtp_server pro1.mail.ovh.net
|
|
smtp_connect_timeout 30
|
|
router_id docker_ingress
|
|
vrrp_skip_check_adv_addr
|
|
vrrp_strict
|
|
vrrp_garp_interval 0
|
|
vrrp_gna_interval 0
|
|
}
|
|
|
|
vrrp_instance VI_1 {
|
|
state MASTER
|
|
interface end0
|
|
virtual_router_id 51
|
|
priority 100
|
|
advert_int 1
|
|
authentication {
|
|
auth_type PASS
|
|
auth_pass transatlantique
|
|
}
|
|
virtual_ipaddress {
|
|
192.168.1.21
|
|
}
|
|
}
|
|
```
|
|
|
|
### On Nodes
|
|
|
|
#### Create/Edit
|
|
|
|
```bash
|
|
sudo vim /etc/keepalived/keepalived.conf
|
|
```
|
|
|
|
```
|
|
! Configuration File for keepalived
|
|
|
|
global_defs {
|
|
notification_email {
|
|
admin@delmar.bzh
|
|
}
|
|
notification_email_from admin@delmar.bzh
|
|
smtp_server pro1.mail.ovh.net
|
|
smtp_connect_timeout 30
|
|
router_id docker_ingress
|
|
vrrp_skip_check_adv_addr
|
|
vrrp_strict
|
|
vrrp_garp_interval 0
|
|
vrrp_gna_interval 0
|
|
}
|
|
|
|
vrrp_instance VI_1 {
|
|
state BACKUP
|
|
interface end0
|
|
virtual_router_id 51
|
|
priority 90
|
|
advert_int 1
|
|
authentication {
|
|
auth_type PASS
|
|
auth_pass transatlantique
|
|
}
|
|
virtual_ipaddress {
|
|
192.168.1.21
|
|
}
|
|
}
|
|
```
|
|
|
|
#### Start/Enable Service
|
|
|
|
```
|
|
sudo systemctl enable keepalived
|
|
sudo systemctl start keepalived
|
|
|
|
ip a show end0
|
|
```
|
|
|
|
---
|
|
|
|
## 4. Glusterfs
|
|
|
|
> *Distributed Storage : install on all nodes*
|
|
|
|
```
|
|
sudo apt-get -y install glusterfs-server
|
|
|
|
sudo systemctl enable glusterd
|
|
sudo systemctl start glusterd
|
|
```
|
|
|
|
> sudo fdisk -l
|
|
|
|
```
|
|
# Format your disk (If you decided to use your internal storage, just ignore the mount and format steps)
|
|
sudo mkfs.xfs -f /dev/nvme0n1
|
|
|
|
# Create brick folder location
|
|
sudo mkdir -p /mnt/ssd
|
|
|
|
# Add line to /etc/fstab to auto mount this disk to /data on boot
|
|
echo "UUID=... /mnt/ssd xfs defaults 0 0" | sudo tee -a /etc/fstab
|
|
|
|
# Create brick folder
|
|
sudo mkdir -p /mnt/ssd/brick
|
|
|
|
# Mount
|
|
sudo mount -a
|
|
|
|
sudo systemctl daemon-reload
|
|
|
|
sudo gluster peer probe gary
|
|
sudo gluster peer probe sheldon
|
|
sudo gluster peer probe pearl
|
|
```
|
|
|
|
> Check
|
|
|
|
```bash
|
|
df -h /mnt/ssd/brick
|
|
|
|
sudo gluster pool list
|
|
```
|
|
```
|
|
UUID Hostname State
|
|
a40ae0e1-90ec-4ccb-88eb-0a13c6c0de26 gary Connected
|
|
fc215f6f-38df-4cce-a0d3-5297e0290d3f sheldon Connected
|
|
13233744-1367-4e76-b486-a0e0b8a190e7 pearl Connected
|
|
07702514-efbc-4ca6-950d-d9c0b14ff61c localhost Connected
|
|
```
|
|
|
|
```bash
|
|
sudo gluster peer status
|
|
```
|
|
```
|
|
Number of Peers: 3
|
|
|
|
Hostname: gary
|
|
Uuid: a40ae0e1-90ec-4ccb-88eb-0a13c6c0de26
|
|
State: Peer in Cluster (Connected)
|
|
|
|
Hostname: sheldon
|
|
Uuid: fc215f6f-38df-4cce-a0d3-5297e0290d3f
|
|
State: Peer in Cluster (Connected)
|
|
|
|
Hostname: pearl
|
|
Uuid: 13233744-1367-4e76-b486-a0e0b8a190e7
|
|
State: Peer in Cluster (Connected)
|
|
|
|
|
|
> For 2 nodes: sandy and sheldon for example
|
|
|
|
```bash
|
|
sudo gluster volume create docker-volume replica 2 transport tcp sandy:/mnt/ssd sheldon:/mnt/ssd force
|
|
```
|
|
|
|
> For all nodes: sandy, gary, sheldon and pearl for example
|
|
|
|
```bash
|
|
sudo gluster volume create dockervolume disperse 4 redundancy 1 sandy:/mnt/ssd/brick gary:/mnt/ssd/brick sheldon:/mnt/ssd/brick pearl:/mnt/ssd/brick force
|
|
```
|
|
|
|
#### Start Gluster Volume
|
|
|
|
```
|
|
sudo gluster volume info dockervolume
|
|
|
|
sudo gluster volume start dockervolume
|
|
```
|
|
|
|
> On all nodes :
|
|
|
|
```
|
|
sudo mkdir /mnt/ssd
|
|
|
|
echo "sandy:/dockervolume /mnt/ssd/brick glusterfs defaults,_netdev 0 0" | sudo tee -a /etc/fstab
|
|
|
|
sudo mount -a
|
|
|
|
sudo systemctl daemon-reload
|
|
```
|
|
|
|
> Check
|
|
|
|
> > Persistent shared storage will be "/mnt/ssd/brick"
|
|
|
|
```
|
|
df -h /mnt/ssd/brick
|
|
|
|
sudo gluster volume set dockervolume auth.allow 127.0.0.1,192.168.1.14,192.168.1.13,192.168.1.16,192.168.1.19
|
|
```
|
|
|
|
> When adding a new brick:
|
|
|
|
```
|
|
sudo gluster volume add-brick **volume** replica X **server:brick**
|
|
```
|
|
|
|
> Option
|
|
|
|
```
|
|
docker plugin install --alias glusterfs mochoa/glusterfs-volume-plugin --grant-all-permissions --disable
|
|
docker plugin set glusterfs SERVERS=192.168.1.14,192.168.1.13,192.168.1.16,192.168.1.19
|
|
docker plugin enable glusterfs
|
|
|
|
docker plugin install --alias glusterfs chrisbecke/glusterfs-volume GFS_VOLUME=dockervolume GFS_SERVERS=192.168.1.14,192.168.1.13,192.168.1.16,192.168.1.19
|
|
docker plugin enable glusterfs
|
|
```
|
|
|
|
# Use in a stack file
|
|
|
|
```ỳaml
|
|
services:
|
|
app:
|
|
image: myapp/web:latest
|
|
volumes:
|
|
- app_data:/app/data
|
|
|
|
volumes:
|
|
app_data:
|
|
driver: trajano/glusterfs-volume-plugin
|
|
driver_opts:
|
|
glusteropts: "--volfile-server=sandy --volfile-id=dockervolume"
|
|
subdir: "app_data"
|
|
```
|
|
|
|
### 5. data-dir
|
|
|
|
Relocate /var/lib/docker
|
|
|
|
```bash
|
|
#Stop docker
|
|
sudo systemctl stop docker.service
|
|
sudo systemctl stop docker.socket
|
|
|
|
# Move data to SSD
|
|
sudo rsync -aP /var/lib/docker/ /mnt/ssd/brick/
|
|
|
|
# Edit docker config
|
|
sudo vim /etc/docker/daemon.json
|
|
```
|
|
|
|
```
|
|
{
|
|
"data-root": "/mnt/ssd/brick",
|
|
"log-driver": "local",
|
|
"log-opts": {
|
|
"max-size": "10m",
|
|
"max-file": "3"
|
|
},
|
|
"dns": ["8.8.8.8", "8.8.4.4"]
|
|
}
|
|
```
|
|
|
|
```bash
|
|
# Restart docker
|
|
sudo systemctl start docker.socket
|
|
sudo systemctl start docker.service
|
|
|
|
# Verify...
|
|
sudo systemctl status docker
|
|
sudo docker info | grep "Docker Root Dir"
|
|
|
|
# Delete old data folder
|
|
sudo rm -rf /var/lib/docker/
|
|
```
|