ADR Examples

Accepted Trade-offs: The team needs to build PostgreSQL-specific operational expertise: query optimisation with EXPLAIN ANALYZE, VACUUM tuning, connection pool management with PgBouncer. Self-managed Patroni HA requires operational investment. This is mitigated by a plan to migrate to Amazon RDS or Cloud SQL (managed) within 12 months of initial deployment.

# High Availability: Deploy Patroni for self-managed HA
# Patroni manages leader election and automatic failover via etcd consensus

# Recommended patroni.yml snippet:
bootstrap:
  dcs:
    ttl: 30
    loop_wait: 10
    retry_timeout: 10
    maximum_lag_on_failover: 1048576  # 1MB
  pg_hba:
    - host replication replicator 0.0.0.0/0 md5
    - host all all 0.0.0.0/0 md5

postgresql:
  listen: 0.0.0.0:5432
  connect_address: "${PATRONI_POSTGRESQL_CONNECT_ADDRESS}:5432"
  data_dir: /data/patroni
  parameters:
    wal_level: replica
    hot_standby: "on"
    max_wal_senders: 5
    max_replication_slots: 5
    wal_log_hints: "on"

# Connection pooling: PgBouncer in transaction mode
[pgbouncer]
listen_port = 6432
listen_addr = *
auth_type = md5
pool_mode = transaction
max_client_conn = 1000
default_pool_size = 25

Accepted Trade-offs: HCL is not a full programming language — complex conditional logic and loops require non-obvious constructs (for_each, dynamic blocks). State management requires discipline: the state file must never be edited manually, and state migrations require care. All engineers must be trained on terraform plan review before being granted apply permissions. Remote state backend must be bootstrapped manually before Terraform can manage itself.

# Remote state backend configuration (AWS)
# backend.tf
terraform {
  backend "s3" {
    bucket         = "company-terraform-state"
    key            = "prod/us-east-1/eks/terraform.tfstate"
    region         = "us-east-1"
    encrypt        = true
    kms_key_id     = "arn:aws:kms:us-east-1:123456789:key/xxxxxxxx"
    dynamodb_table = "terraform-state-lock"
  }
  required_providers {
    aws = { source = "hashicorp/aws", version = "~> 5.0" }
    google = { source = "hashicorp/google", version = "~> 5.0" }
  }
  required_version = ">= 1.6.0"
}

# Remote state backend configuration (GCP)
# backend-gcp.tf
terraform {
  backend "gcs" {
    bucket = "company-terraform-state-gcp"
    prefix = "prod/europe-west1/gke"
  }
}

# Workspace strategy for environment isolation
# Use separate state files per environment, not workspaces, for blast-radius isolation
# Structure:
# environments/
#   prod/
#     us-east-1/
#       vpc/          → terraform.tfstate in s3://.../prod/us-east-1/vpc/
#       eks/          → terraform.tfstate in s3://.../prod/us-east-1/eks/
#       rds/          → terraform.tfstate in s3://.../prod/us-east-1/rds/
#   staging/
#     us-east-1/
#       vpc/
#       eks/

Accepted Trade-offs: All Kubernetes manifests (or Helm charts / Kustomize overlays) must live in a Git repository accessible to ArgoCD — no more ad-hoc kubectl apply. This introduces process discipline that some engineers will initially resist. An ArgoCD cluster is itself required to be running and healthy for deployments to function — this adds an operational dependency. ArgoCD itself must be bootstrapped manually (or via Terraform) the first time.

# App of Apps pattern — root ArgoCD Application manages all other Applications
# File: argocd/root-app.yaml (in the gitops-config repository)

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: root-app
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://github.com/company/gitops-config.git
    targetRevision: main
    path: apps/prod
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true

---
# ApplicationSet for multi-environment deployment
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: microservices
  namespace: argocd
spec:
  generators:
  - matrix:
      generators:
      - git:
          repoURL: https://github.com/company/gitops-config.git
          revision: main
          directories:
          - path: services/*
      - list:
          elements:
          - cluster: prod-us-east-1
            url: https://prod-us-east-1.example.com
          - cluster: staging-us-east-1
            url: https://staging-us-east-1.example.com
  template:
    metadata:
      name: '{{path.basename}}-{{cluster}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/company/gitops-config.git
        targetRevision: main
        path: '{{path}}/overlays/{{cluster}}'
      destination:
        server: '{{url}}'
        namespace: '{{path.basename}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true

Accepted Trade-offs: Running Vault adds operational overhead — the Vault cluster (3-node HA with Raft storage) must be maintained, upgraded, and monitored. The Vault seal/unseal lifecycle requires careful key management (we use AWS KMS auto-unseal). Vault itself must be excluded from the secrets it manages. Engineers must be trained on Vault policies, AppRole authentication, and the Kubernetes auth method. On-call engineers must be prepared to handle Vault availability incidents as a Tier 1 dependency.

# Deploy Vault on Kubernetes using official Helm chart
helm repo add hashicorp https://helm.releases.hashicorp.com

# values.yaml for production HA deployment
cat > vault-values.yaml <<'EOF'
server:
  ha:
    enabled: true
    replicas: 3
    raft:
      enabled: true
      setNodeId: true
      config: |
        cluster_name = "production"
        storage "raft" {
          path = "/vault/data"
          retry_join {
            leader_api_addr = "http://vault-0.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-1.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-2.vault-internal:8200"
          }
        }
        seal "awskms" {
          region     = "us-east-1"
          kms_key_id = "arn:aws:kms:us-east-1:123456789:key/xxxxxxxx"
        }
        listener "tcp" {
          tls_disable = false
          tls_cert_file = "/vault/userconfig/vault-tls/tls.crt"
          tls_key_file  = "/vault/userconfig/vault-tls/tls.key"
        }
        api_addr = "https://vault.vault.svc.cluster.local:8200"
        cluster_addr = "https://POD_IP:8201"
  auditStorage:
    enabled: true
    size: 10Gi
injector:
  enabled: true
  replicas: 2
EOF

helm install vault hashicorp/vault \
  -n vault --create-namespace \
  -f vault-values.yaml

# Enable Kubernetes auth method
vault auth enable kubernetes
vault write auth/kubernetes/config \
  kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443"

# Example Vault Agent Injector annotations on a Pod
# The sidecar reads the Vault path and writes the secret to /vault/secrets/
metadata:
  annotations:
    vault.hashicorp.com/agent-inject: "true"
    vault.hashicorp.com/role: "myapp-prod"
    vault.hashicorp.com/agent-inject-secret-db-creds: "database/creds/myapp-prod"
    vault.hashicorp.com/agent-inject-template-db-creds: |
      {{- with secret "database/creds/myapp-prod" -}}
      DB_USERNAME={{ .Data.username }}
      DB_PASSWORD={{ .Data.password }}
      {{- end -}}