Docker Based Server Setup

This document describes the steps to create a Docker based server environment on a Rocky Linux 9 server.

Some initial housekeeping

sudo dnf install vim -y
sudo visudo
    > Defaults    env_reset,timestamp_timeout=30
sudo timedatectl set-timezone America/New_York
sudo dnf install -y chrony
sudo dnf -y update > dnf-update-[date].txt
sudo shutdown -r now

Set up firewall

see Firewall: allow certain access

Install and start docker

see Install and start Docker

Create application user and set them up for docker

sudo adduser -u 5678 appuser
sudo vi /home/appuser/.bashrc
    > export HISTTIMEFORMAT="%Y-%m-%d %H:%M "
(appuser) mkdir .ssh
(appuser) touch .ssh/authorized_keys
(appuser) chmod 600 .ssh/authorized_keys
sudo usermod -aG docker appuser

Set up rsnapshot backups

Create the following files on the server, in /home/appuser/rsnapshot.

Note the backups for /etc, /root, /home, and /usr/local are stored in directories annotated with -host. This allows the backup of the host without overwriting the container folders of the same name.

In this example, the backups are stored at /mnt/backup/snapshots. Be sure to chmod this directory to 700 so that only root can read the backup data.

docker-compose.yml

# see https://hub.docker.com/r/linuxserver/rsnapshot
services:
  rsnapshot:
    image: lscr.io/linuxserver/rsnapshot:latest
    container_name: rsnapshot
    environment:
      - PUID=0 #root
      - PGID=0 #root
      - TZ=America/New_York
    volumes:
      - ./config:/config
      - /mnt/backup/snapshots:/.snapshots
      - ./custom-init:/custom-cont-init.d:ro
      - /home:/home-host
      - ./msmtprc:/etc/msmtprc:ro
      - /etc:/etc-host
      - /usr/local:/usr/local-host
      - /root:/root-host
    restart: unless-stopped

config/crontabs/root

# The values used correspond to the examples in /etc/rsnapshot.conf.
# There you can also set the backup points and many other things.
#
# To activate this cron file you have to uncomment the lines below.
# Feel free to adapt it to your needs.

0 */4 * * * /usr/bin/rsnapshot hourly
30 3  * * * /usr/bin/rsnapshot daily
0  3  * * 1 /usr/bin/rsnapshot weekly
30 2  1 * * /usr/bin/rsnapshot monthly

config/rsnapshot.conf

#################################################
# rsnapshot.conf - rsnapshot configuration file #
#################################################
#                                               #
# PLEASE BE AWARE OF THE FOLLOWING RULE:        #
#                                               #
# This file requires tabs between elements      #
#                                               #
#################################################

#######################
# CONFIG FILE VERSION #
#######################

config_version	1.2

###########################
# SNAPSHOT ROOT DIRECTORY #
###########################

# All snapshots will be stored under this root directory.
#
snapshot_root	/.snapshots/

# If no_create_root is enabled, rsnapshot will not automatically create the
# snapshot_root directory. This is particularly useful if you are backing
# up to removable media, such as a FireWire or USB drive.
#
#no_create_root	1

#################################
# EXTERNAL PROGRAM DEPENDENCIES #
#################################

# LINUX USERS:   Be sure to uncomment "cmd_cp". This gives you extra features.
# EVERYONE ELSE: Leave "cmd_cp" commented out for compatibility.
#
# See the README file or the man page for more details.
#
cmd_cp		/bin/cp

# uncomment this to use the rm program instead of the built-in perl routine.
#
cmd_rm		/bin/rm

# rsync must be enabled for anything to work. This is the only command that
# must be enabled.
#
cmd_rsync	/usr/bin/rsync

# Uncomment this to enable remote ssh backups over rsync.
#
cmd_ssh	/usr/bin/ssh

# Comment this out to disable syslog support.
#
cmd_logger	/usr/bin/logger

# Uncomment this to specify the path to "du" for disk usage checks.
# If you have an older version of "du", you may also want to check the
# "du_args" parameter below.
#
cmd_du		/usr/bin/du

# Uncomment this to specify the path to rsnapshot-diff.
#
#cmd_rsnapshot_diff	/usr/bin/rsnapshot-diff

# Specify the path to a script (and any optional arguments) to run right
# before rsnapshot syncs files
#
#cmd_preexec	/path/to/preexec/script

# Specify the path to a script (and any optional arguments) to run right
# after rsnapshot syncs files
#
#cmd_postexec	/path/to/postexec/script

# Paths to lvcreate, lvremove, mount and umount commands, for use with
# Linux LVMs.
#
#linux_lvm_cmd_lvcreate	/path/to/lvcreate
#linux_lvm_cmd_lvremove	/path/to/lvremove
#linux_lvm_cmd_mount	/bin/mount
#linux_lvm_cmd_umount	/bin/umount

#########################################
#     BACKUP LEVELS / INTERVALS         #
# Must be unique and in ascending order #
# e.g. alpha, beta, gamma, etc.         #
#########################################

#retain	alpha	6
#retain	beta	7
#retain	gamma	4
#retain	delta	3

retain	hourly	6
retain	daily	7
retain	weekly	4
retain	monthly	3

############################################
#              GLOBAL OPTIONS              #
# All are optional, with sensible defaults #
############################################

# Verbose level, 1 through 5.
# 1     Quiet           Print fatal errors only
# 2     Default         Print errors and warnings only
# 3     Verbose         Show equivalent shell commands being executed
# 4     Extra Verbose   Show extra verbose information
# 5     Debug mode      Everything
#
verbose		2

# Same as "verbose" above, but controls the amount of data sent to the
# logfile, if one is being used. The default is 3.
#
loglevel	3

# If you enable this, data will be written to the file you specify. The
# amount of data written is controlled by the "loglevel" parameter.
#
logfile	/config/rsnapshot.log

# If enabled, rsnapshot will write a lockfile to prevent two instances
# from running simultaneously (and messing up the snapshot_root).
# If you enable this, make sure the lockfile directory is not world
# writable. Otherwise anyone can prevent the program from running.
#
lockfile	/config/rsnapshot.pid

# By default, rsnapshot check lockfile, check if PID is running
# and if not, consider lockfile as stale, then start
# Enabling this stop rsnapshot if PID in lockfile is not running
#
#stop_on_stale_lockfile		0

# Default rsync args. All rsync commands have at least these options set.
#
#rsync_short_args	-a
#rsync_long_args	--delete --numeric-ids --relative --delete-excluded

# ssh has no args passed by default, but you can specify some here.
#
#ssh_args	-p 22

# Default arguments for the "du" program (for disk space reporting).
# The GNU version of "du" is preferred. See the man page for more details.
# If your version of "du" doesn't support the -h flag, try -k flag instead.
#
#du_args	-csh

# If this is enabled, rsync won't span filesystem partitions within a
# backup point. This essentially passes the -x option to rsync.
# The default is 0 (off).
#
#one_fs		0

# The include and exclude parameters, if enabled, simply get passed directly
# to rsync. If you have multiple include/exclude patterns, put each one on a
# separate line. Please look up the --include and --exclude options in the
# rsync man page for more details on how to specify file name patterns.
#
#include	???
#include	???
#exclude	???
#exclude	???

# The include_file and exclude_file parameters, if enabled, simply get
# passed directly to rsync. Please look up the --include-from and
# --exclude-from options in the rsync man page for more details.
#
#include_file	/path/to/include/file
#exclude_file	/path/to/exclude/file

# If your version of rsync supports --link-dest, consider enabling this.
# This is the best way to support special files (FIFOs, etc) cross-platform.
# The default is 0 (off).
#
link_dest	1

# When sync_first is enabled, it changes the default behaviour of rsnapshot.
# Normally, when rsnapshot is called with its lowest interval
# (i.e.: "rsnapshot alpha"), it will sync files AND rotate the lowest
# intervals. With sync_first enabled, "rsnapshot sync" handles the file sync,
# and all interval calls simply rotate files. See the man page for more
# details. The default is 0 (off).
#
#sync_first	0

# If enabled, rsnapshot will move the oldest directory for each interval
# to [interval_name].delete, then it will remove the lockfile and delete
# that directory just before it exits. The default is 0 (off).
#
#use_lazy_deletes	0

# Number of rsync re-tries. If you experience any network problems or
# network card issues that tend to cause ssh to fail with errors like
# "Corrupted MAC on input", for example, set this to a non-zero value
# to have the rsync operation re-tried.
#
#rsync_numtries 0

# LVM parameters. Used to backup with creating lvm snapshot before backup
# and removing it after. This should ensure consistency of data in some special
# cases
#
# LVM snapshot(s) size (lvcreate --size option).
#
#linux_lvm_snapshotsize	100M

# Name to be used when creating the LVM logical volume snapshot(s).
#
#linux_lvm_snapshotname	rsnapshot

# Path to the LVM Volume Groups.
#
#linux_lvm_vgpath	/dev

# Mount point to use to temporarily mount the snapshot(s).
#
#linux_lvm_mountpath	/path/to/mount/lvm/snapshot/during/backup

###############################
### BACKUP POINTS / SCRIPTS ###
###############################

# LOCALHOST
#backup	/data/		localhost/

backup	/home-host/		localhost/
backup	/etc-host/		localhost/
backup	/usr/local-host/	localhost/
backup	/config/rsnapshot.log		localhost/
#backup	/var/www	localhost/
backup	/root-host	localhost/
#backup	/etc/passwd-host	localhost/

#backup	/home/foo/My Documents/		localhost/
#backup	/foo/bar/	localhost/	one_fs=1,rsync_short_args=-urltvpog
#backup_script	/usr/local/bin/backup_pgsql.sh	localhost/postgres/
# You must set linux_lvm_* parameters below before using lvm snapshots
#backup	lvm://vg0/xen-home/	lvm-vg0/xen-home/

# EXAMPLE.COM
#backup_exec	/bin/date "+ backup of example.com started at %c"
#backup	root@example.com:/home/	example.com/	+rsync_long_args=--bwlimit=16,exclude=core
#backup	root@example.com:/etc/	example.com/	exclude=mtab,exclude=core
#backup_exec	ssh root@example.com "mysqldump -A > /var/db/dump/mysql.sql"
#backup	root@example.com:/var/db/dump/	example.com/
#backup_exec	/bin/date "+ backup of example.com ended at %c"

# CVS.SOURCEFORGE.NET
#backup_script	/usr/local/bin/backup_rsnapshot_cvsroot.sh	rsnapshot.cvs.sourceforge.net/

# RSYNC.SAMBA.ORG
#backup	rsync://rsync.samba.org/rsyncftp/	rsync.samba.org/rsyncftp/

custom-init/rsnapshot-init

#!/bin/bash

# use msmtp for sendmail, need volume with /etc/msmtprc defined
apk add --no-cache msmtp && rm /usr/sbin/sendmail && ln -s /usr/bin/msmtp /usr/sbin/sendmail

From this directory, start the service with docker compose up -d

Set up caddy

Create the following files on the server, in /home/appuser/caddy

Note in Caddyfile, logging can’t use {common_log} because of the reverse proxy setup with Cloudflare. Instead, a custom log format is used that checks for the X-Forwarded-For header first. (see https://github.com/caddyserver/transform-encoder, https://caddyserver.com/docs/caddyfile/matchers, and https://developers.cloudflare.com/support/troubleshooting/restoring-visitor-ips/restoring-original-visitor-ips/)

Note in Dockerfile, caddy must be built with caddy-dns/cloudflare to be able to update Cloudflare DNS records for automatic https certificate management, and with WeidiDeng/caddy-cloudflare-ip to periodically update the list of Cloudflare IPs for correct client IP logging behind the Cloudflare reverse proxy. (see https://github.com/caddy-dns/cloudflare, https://github.com/WeidiDeng/caddy-cloudflare-ip, https://roelofjanelsinga.com/articles/using-caddy-ssl-with-cloudflare/, and https://caddy.community/t/trusted-proxies-with-cloudflare-my-solution/16124/6 )

Note in docker-compose.yml the caddy-backend-network is shared with the webapp containers to allow caddy to reverse proxy for the webapps.

Note in Caddyfile, the reverse_proxy port must match the webapp port.

.env

APP_VER=[version number]
CF_API_TOKEN=[cloudflare api token]

docker-compose.yml

services:
  caddy:
    image: louking/caddy:${APP_VER}
    build:
      dockerfile: Dockerfile
    restart: unless-stopped
    cap_add:
      - NET_ADMIN
    ports:
      - "80:80"
      - "443:443"
      - "443:443/udp"
    volumes:
      - $PWD/config:/etc/caddy
      - $PWD/logs:/var/log/caddy
      - caddy_data:/data
      - caddy_config:/config
    environment:
      - CF_API_TOKEN=${CF_API_TOKEN}
    extra_hosts:
      # see https://stackoverflow.com/a/67158212/799921
      - host.docker.internal:host-gateway
    networks:
      - backend-network

networks:
  backend-network:
    name: caddy-backend-network
    external: true

volumes:
  caddy_data:
  caddy_config:

Dockerfile

FROM caddy:builder-alpine AS builder

# see https://caddyserver.com/docs/modules/http.ip_sources.cloudflare
RUN xcaddy build \
    --with github.com/caddyserver/transform-encoder \
    --with github.com/caddy-dns/cloudflare \
    --with github.com/WeidiDeng/caddy-cloudflare-ip

FROM caddy:alpine

COPY --from=builder /usr/bin/caddy /usr/bin/caddy

config/Caddyfile

(proxy_log_format) {
    # need to use X-Forwarded-For if it exists -- see https://caddyserver.com/docs/caddyfile/matchers (remote_ip) and https://developers.cloudflare.com/support/troubleshooting/restoring-visitor-ips/restoring-original-visitor-ips/
    format transform `{request>headers>X-Forwarded-For>[0]:request>remote_ip} - {user_id} [{ts}] "{request>method} {request>uri} {request>proto}" {status} {size} "{request>headers>Referer>[0]}" "{request>headers>User-Agent>[0]}"` {
        time_format "02/Jan/2006:15:04:05 -0700"
    }
}

(proxy_log_rotate) {
    roll_size 10MB    # Rotate when log file reaches 10MB
    roll_keep 2       # Keep 2 rotated log files
    roll_keep_for 168h  # Keep rotated files for 7 days
}

{
    servers {
        # https://caddy.community/t/trusted-proxies-with-cloudflare-my-solution/16124/6
        trusted_proxies cloudflare {
            interval 12h
            timeout 15s
        }

        client_ip_headers Cf-Connecting-Ip X-Forwarded-For X-Real-IP
    }

    log default {
        format console
        output file /var/log/caddy/caddy.log
    }
}

weewx2.lousbrews.info {
    tls {
        dns cloudflare {env.CF_API_TOKEN}
    }
    log {
        import proxy_log_format
        output file /var/log/caddy/weewx.lousbrews.info/access.log {
            import proxy_log_rotate
        }
    }

    reverse_proxy host.docker.internal:8080
}

# Catch-all site block for unknown hosts
# This address (https://) is very unspecific and will catch requests not matched
# by the above hostnames on the HTTPS port (443).
# see https://gemini.google.com/share/4775da3f299e
https:// { 
    # Log the request before rejecting
    log {
        import proxy_log_format
        output file /var/log/caddy/unknown_hosts/access.log
    }
    
    # Reject the request with a 403 Forbidden status
    respond "Access Denied: Unknown Host" 403
}

From this directory, start the service with docker compose up -d

Set up fail2ban

Create the following files on the server, in /home/appuser/fail2ban

.env

USER_ID=[fail2ban user id]
GROUP_ID=[fail2ban user group id]

docker-compose.yml

# see https://hub.docker.com/r/linuxserver/fail2ban

services:
  fail2ban:
    image: lscr.io/linuxserver/fail2ban:latest
    container_name: fail2ban
    cap_add:
      - NET_ADMIN
      - NET_RAW
    network_mode: host
    environment:
      - PUID=${USER_ID}
      - PGID=${GROUP_ID}
      - TZ=America/New_York
    volumes:
      - $PWD/config:/config
      - /var/log:/var/log:ro
      - ../caddy/logs:/remotelogs/apache2:ro #optional
    restart: unless-stopped

config/fail2ban/jail.local

#

[DEFAULT]

destemail = [email for errors]

bantime = 1h

# Default banning action (e.g. iptables, iptables-new,
# iptables-multiport, shorewall, etc) It is used to define
# action_* variables. Can be overridden globally or per
# section within jail.local file
banaction = iptables-multiport
banaction_allports = iptables-allports

[sshd]
enabled = true

[apache-auth]
enabled = false

[apache-badbots]
enabled = true

[apache-noscript]
enabled = false

[apache-overflows]
enabled = false

[apache-nohome]
enabled = false

[apache-botsearch]
enabled = false

[apache-fakegooglebot]
enabled = true

[apache-modsecurity]
enabled = false

[apache-shellshock]
enabled = false

[caddyaccess]
enabled = true
filter = caddy-access
logpath = %(apache_access_log)s
port = http,https
maxretry = 5
findtime = 30
bantime = 600

config/fail2ban/paths-overrides.local

[DEFAULT]

apache_error_log = /remotelogs/apache2/*/error.log
apache_access_log = /remotelogs/apache2/*/access.log

syslog_authpriv = %(var_log_path)s/secure

config/fail2ban/filter.d/caddy-access.local

# from https://www.ottorask.com/blog/caddy-and-fail2ban

[Definition]
failregex = ^<HOST>.*"(GET|POST|OPTION).*" (4[0-9][0-9])[ \d]*$
ignoreregex =

From this directory, start the service with docker compose up -d

Set up web site

Create the following files on the server, in /home/appuser/[website]

In this example, static website files are stored in /home/appuser/webapp/html

template files can be stored in /home/appuser/webapp/templates. See https://hub.docker.com/_/nginx for details on using nginx with templates.

docker-compose.yml

services:
  web:
    image: nginx:mainline-alpine
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ./templates:/etc/nginx/templates:ro
      - ./html:/usr/share/nginx/html:ro
    ports:
      - "8080:80"
    networks:
      - backend-network

networks:
  backend-network:
    name: caddy-backend-network
    external: true

nginx.conf

events { }
# uncomment to debug
# error_log /var/log/nginx/error.log debug;

http {
    include mime.types;

    # see https://github.com/sjmf/reverse-proxy-minimal-example
    map $http_x_forwarded_proto $real_scheme {
        default $scheme;
        https "https";
    }

    map $http_host $port {
        default $server_port;
        "~^[^\:]+:(?<p>\d+)$" $p;
    }

    server {
        listen       80;
        root /usr/share/nginx/html;

        # see https://developers.cloudflare.com/support/troubleshooting/restoring-visitor-ips/restoring-original-visitor-ips/#web-server-instructions
        set_real_ip_from 0.0.0.0/32;

        real_ip_header X-Forwarded-For;

	# see https://stackoverflow.com/questions/55691000/how-to-include-location-blocks-in-nginx
        # simple reverse-proxy
        # pass requests for dynamic content to the Flask server
        location / {
            # see https://github.com/sjmf/reverse-proxy-minimal-example
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Host $host;
            proxy_set_header X-Forwarded-Proto $real_scheme;

            # see https://computingforgeeks.com/how-to-solve-nginx-warn-could-not-build-optimal-proxy-headers-hash-error/
            proxy_headers_hash_max_size 512;
            proxy_headers_hash_bucket_size 128; 

        }

    }

    include /etc/nginx/conf.d/*.conf;
}

html/index.html

<p>hello world</p>

From this directory, start the service with docker compose up -d