Docker Swarm for Orchestration

# Initialize a new swarm (on manager node)
docker swarm init --advertise-addr <MANAGER-IP>
# --advertise-addr specifies the address other nodes will use to connect to this manager
# If omitted, Docker will attempt to auto-detect the IP, which may be incorrect in multi-interface servers

# Output will provide a token to join worker nodes
# Example output:
# docker swarm join --token SWMTKN-1-49nj1cmql0jkz5s954yi3oex3nedyz0fb0xx14ie39trti4wxv-8vxv8rssmk743ojnwacrr2e7c <MANAGER-IP>:2377
# This token authenticates new nodes to the swarm, ensuring secure cluster expansion

# Join a node as worker
docker swarm join --token <TOKEN> <MANAGER-IP>:2377
# Workers receive and execute tasks but cannot manage the cluster state
# Port 2377 is the standard swarm management port

# Join a node as manager
docker swarm join-token manager
# This command generates a different token specifically for manager nodes
# Then use the provided token to join as manager
docker swarm join --token <MANAGER-TOKEN> <MANAGER-IP>:2377
# Manager nodes participate in the Raft consensus and can manage the cluster

# List nodes in the swarm
docker node ls
# Shows all nodes with their roles, availability status, and manager status
# MANAGER STATUS column shows "Leader" for the primary manager node

# Create a replicated service
docker service create --name webserver \
  --replicas 3 \                        # Deploy 3 instances of the container
  --publish 80:80 \                     # Publish port 80 on all swarm nodes
  --update-delay 10s \                  # Wait 10s between updating each container
  --update-parallelism 1 \              # Update one container at a time
  --restart-condition on-failure \      # Automatically restart containers that exit non-zero
  nginx:latest                          # Container image to use

# List services
docker service ls
# Shows all services with their ID, name, mode (replicated/global), replicas, image, and ports

# Inspect a service
docker service inspect webserver
# Provides detailed JSON output of the service configuration, including:
# - Container configuration
# - Resource constraints
# - Networks
# - Update and rollback configuration
# - Placement constraints

# View service tasks (containers)
docker service ps webserver
# Shows all tasks (containers) for the service with their:
# - ID, name, image
# - Node assignment
# - Desired and current state
# - Error messages (if any)
# - When the task was created and updated

# Scale a service
docker service scale webserver=5
# Increases or decreases the number of replicas
# Swarm will create or remove containers to match the desired count
# Use comma-separated list to scale multiple services: webserver=5,api=3

# Update a service
docker service update --image nginx:1.21 \  # Change the image version
  --limit-cpu 0.5 \                         # Add CPU limits
  --limit-memory 512M \                     # Add memory limits  
  --rollback-parallelism 2 \                # Configure rollback behavior
  webserver

# Remove a service
docker service rm webserver
# Removes the service and stops all associated containers

# Service discovery example
docker service create --name db \
  --network backend-network \               # Join the overlay network
  --mount type=volume,source=db-data,target=/var/lib/postgresql/data \  # Persistent storage
  --env POSTGRES_PASSWORD_FILE=/run/secrets/db_password \  # Use secrets for security
  --replicas 1 \                            # Database typically runs a single instance
  postgres:13                               # Database image

docker service create --name api \
  --network backend-network \               # Same network as the db service
  --env DB_HOST=db \                        # Reference the database by service name
  --env DB_PORT=5432 \                      # Standard PostgreSQL port
  --replicas 3 \                            # Run multiple API instances
  --update-order start-first \              # Start new tasks before stopping old ones during updates
  myapi:latest                              # API image

# Connect to database from inside the api container
docker exec -it $(docker ps -q -f name=api) bash
ping db       # Resolves to the service VIP
psql -h db -U postgres -d myapp  # Connect to database using service name

# Create frontend service that connects to the API
docker service create --name frontend \
  --network backend-network \
  --env API_URL=http://api:8000/ \  # API service name is automatically resolved and load-balanced
  --publish 80:80 \
  frontend:latest

# docker-compose.yml for Swarm deployment
version: '3.8'  # Compose file format version with swarm support

services:
  web:
    image: nginx:latest
    ports:
      - "80:80"  # Published port (accessible from outside)
    deploy:
      mode: replicated       # 'replicated' (scaled) or 'global' (one per node)
      replicas: 3            # Number of container instances
      update_config:
        parallelism: 1       # Update one container at a time
        delay: 10s           # Wait 10s between updates
        order: start-first   # Start new tasks before stopping old ones
        failure_action: rollback  # Auto-rollback on failure
      restart_policy:
        condition: on-failure    # Restart if container exits non-zero
        max_attempts: 3          # Maximum restart attempts
        window: 120s             # Time window to evaluate restart attempts
      placement:
        constraints:
          - node.role == worker  # Only run on worker nodes
        preferences:
          - spread: node.labels.datacenter  # Spread across datacenters
      resources:
        limits:
          cpus: '0.5'       # Maximum CPU usage
          memory: 512M      # Maximum memory usage
        reservations:
          cpus: '0.1'       # Guaranteed CPU allocation
          memory: 128M      # Guaranteed memory allocation
  
  api:
    image: myapi:latest
    deploy:
      replicas: 3
      placement:
        constraints:
          - node.labels.zone == frontend  # Only run on nodes with this label
        max_replicas_per_node: 1          # Limit instances per node for HA
      update_config:
        parallelism: 2                    # Update two containers at once
      rollback_config:
        parallelism: 3                    # Faster rollback if needed
    environment:
      - DB_HOST=db                     # Service discovery by name
      - API_KEY_FILE=/run/secrets/api_key  # Reference a secret
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s                # Initial grace period
    networks:
      - backend                        # Connect to backend network
    secrets:
      - api_key                        # Reference to secret defined below
  
  db:
    image: postgres:13
    volumes:
      - db-data:/var/lib/postgresql/data  # Persistent storage
    deploy:
      placement:
        constraints:
          - node.labels.zone == database  # Only run on database nodes
      replicas: 1                         # Single database instance
      restart_policy:
        condition: any                    # Always restart
    environment:
      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
    networks:
      - backend
    secrets:
      - db_password

networks:
  backend:
    driver: overlay               # Multi-host network
    attachable: true              # Allow standalone containers to connect
    driver_opts:
      encrypted: "true"           # Encrypt traffic on this network

volumes:
  db-data:                        # Named volume for persistent data
    driver: local                 # Use local storage (default)
    # For production, consider using a volume driver that supports replication

secrets:
  api_key:
    file: ./secrets/api_key.txt   # Load from local file during deployment
  db_password:
    file: ./secrets/db_password.txt

# Create a secret
echo "mypassword" | docker secret create db_password -
# Secrets are stored encrypted in the Raft log
# They are never written to disk unencrypted
# `-` at the end indicates input from stdin

# Create a service with a secret
docker service create --name db \
  --secret db_password \                           # Reference the created secret
  --env POSTGRES_PASSWORD_FILE=/run/secrets/db_password \  # Tell application where to find it
  --secret source=ssl_cert,target=server.crt \     # Custom target path for a secret
  --secret source=ssl_key,target=server.key,mode=0400 \  # Custom file permissions
  postgres:13

# Inside the container, secrets appear as files in /run/secrets/
# Example:
# /run/secrets/db_password
# /run/secrets/server.crt
# /run/secrets/server.key

# List secrets
docker secret ls
# Shows all secrets with creation time and ID

# Inspect a secret (shows metadata only, not the actual content)
docker secret inspect db_password

# Remove a secret
docker secret rm db_password
# Note: Cannot remove secrets used by running services
# Must update or remove services first

# Create a secret from a file
docker secret create ssl_cert ./server.crt

# Create a service with resource constraints
docker service create --name resource-limited \
  --limit-cpu 0.5 \                   # Maximum CPU usage (50% of one core)
  --limit-memory 512M \               # Maximum memory usage (512MB)
  --reserve-cpu 0.2 \                 # Guaranteed CPU (20% of one core)
  --reserve-memory 256M \             # Guaranteed memory (256MB)
  --reserve-memory-swap 512M \        # Total memory+swap reservation
  --generic-resource "gpu=1" \        # Request 1 GPU (requires node labels)
  --ulimit nofile=65536:65536 \       # Set file descriptor limits
  --ulimit nproc=4096:4096 \          # Process limits
  nginx:latest

# Resource constraints ensure:
# 1. Fair resource distribution across services
# 2. Protection against noisy neighbors
# 3. Predictable performance
# 4. Efficient bin-packing of containers

# The scheduler uses reservations to make placement decisions
# Limits enforce maximum resource usage
# A node must have available resources matching the reservation

# List nodes
docker node ls
# Output shows:
# - NODE ID: Unique identifier for each node
# - HOSTNAME: Node's hostname
# - STATUS: Ready, Down, or Disconnected
# - AVAILABILITY: Active, Pause, or Drain
# - MANAGER STATUS: Leader, Reachable, or Unreachable (for manager nodes)
# - ENGINE VERSION: Docker engine version

# Inspect node
docker node inspect node-1
# Returns detailed information about the node:
# - Labels and node attributes
# - Resource availability
# - Platform and architecture
# - Network addresses
# - Join tokens and certificates
# - Status and health

# Format inspect output for specific information
docker node inspect -f '{{.Status.Addr}}' node-1
# Returns just the node's IP address

# Set node availability
docker node update --availability drain node-1  # Prepare for maintenance
# Drain: Stops scheduling new tasks and removes existing tasks from the node
# Existing tasks are gracefully rescheduled to other nodes
# Perfect for maintenance operations and updates

docker node update --availability active node-1 # Return to service
# Active: Node accepts new tasks and retains existing tasks
# Pause: Node keeps existing tasks but won't receive new ones

# Add labels to nodes
docker node update --label-add zone=frontend node-1
docker node update --label-add datacenter=east --label-add cpu=high-performance node-1
# Labels enable sophisticated scheduling strategies:
# - Geographic distribution
# - Hardware differentiation
# - Environment separation (prod/staging)
# - Specialized workload targeting

# Filter nodes by labels
docker node ls --filter node.label=zone=frontend

# Promote worker to manager
docker node promote node-2
# Converts worker to manager role
# Joins the Raft consensus group
# Receives a copy of the cluster state
# Can now accept management commands

# Demote manager to worker
docker node demote node-2
# Removes node from management plane
# If this would break quorum, the operation fails
# Best practice: Demote before removing manager nodes

# Remove a node from the swarm
# First, on the node to remove:
docker swarm leave
# Then on a manager node:
docker node rm node-3

# Backup swarm state (on manager node)
# First, create a full backup
tar -czvf swarm-backup.tar.gz /var/lib/docker/swarm
# This includes:
# - Raft logs and consensus data
# - TLS certificates and keys
# - Secret data (encrypted)
# - Node information and join tokens

# For a more comprehensive backup, include:
docker service ls > services.txt
docker service inspect $(docker service ls -q) > service-details.json
docker secret ls > secrets.txt
docker config ls > configs.txt
docker network ls --filter driver=overlay > networks.txt

# Restore swarm
# 1. Stop Docker on the manager
systemctl stop docker

# 2. Restore the backup
tar -xzvf swarm-backup.tar.gz -C /
# Backup should be restored to the same path structure

# 3. Start Docker
systemctl start docker
# Docker will initialize using the restored swarm state
# The node resumes its role (leader or follower)

# 4. Verify the restore
docker node ls
docker service ls

# Alternative backup approach using manager UCP
# Save the UCP backup file which includes swarm configuration
docker container run --log-driver none --rm -i --name ucp \
  -v /var/run/docker.sock:/var/run/docker.sock \
  docker/ucp:latest backup --id $(docker container ls -q --filter name=ucp-controller) \
  --passphrase "secret" > ucp-backup.tar.gz