Nomad monitoring with Thanos

fabio.nomad

job "fabio" {
  datacenters = ["dc1"]
  type = "system"

  group "fabio" {
    network {
      mode = "host"

      port "lb" {
        static = 9999
        to = 9999
      }
      port "ui" {
        static = 9998
        to = 9998
      }
    }

    task "fabio" {
      driver = "docker"
      
      config {
        image = "fabiolb/fabio"
        network_mode = "host"
      }

      resources {
        cpu    = 100
        memory = 64
      }
    }
  }
}

monitoring.nomad

job "monitoring" {
  datacenters = ["dc1"]

  group "prometheus" {
    count = 1

    network {
      mode = "bridge"

      port "prometheus" {
        to = 9090
      }
      port "grpc" {
        to = 10901
      }
      port "http" {
        to = 10902
      }
    }

    service {
      name = "prometheus"
      port = "prometheus"
      tags = ["urlprefix-/prometheus"]

      check {
        name     = "prometheus query port alive"
        type     = "http"
        path     = "/prometheus/-/healthy"
        interval = "10s"
        timeout  = "2s"
      }
    }

    service {
      name = "thanos-sidecar"
      port = "grpc"
      tags = ["grpc"]
    }
    
    service {
      name = "thanos-sidecar"
      port = "http"
      tags = ["http"]
    }

    volume "prometheus" {
      type      = "host"
      read_only = false
      source    = "prometheus"
    }

    task "init" {
      lifecycle {
        hook = "prestart"
      }

      driver = "docker"

      volume_mount {
        volume      = "prometheus"
        destination = "/var/lib/prometheus"
        read_only   = false
      }

      config {
        image   = "debian:9-slim"
        command = "chown"
        args = ["-R", "65534:65534", "/var/lib/prometheus"]
      }
    }

    task "prometheus" {
      template {
        change_mode = "noop"
        destination = "local/prometheus.yml"

        data = <<EOH
---
global:
  scrape_interval:     5s
  evaluation_interval: 5s
  external_labels:
    platform_name: 'orion' 
    platform_type: 'nomad'
scrape_configs:
  - job_name: 'nomad_metrics'
    scrape_interval: 15s
    metrics_path: /v1/metrics
    params:
      format: ['prometheus']
    consul_sd_configs:
    - server: '{{ env "attr.unique.network.ip-address" }}:8500'
      services: ['nomad-client', 'nomad']
    relabel_configs:
    - source_labels: ['__meta_consul_tags']
      regex: '(.*)http(.*)'
      action: keep
EOH
      }

      driver = "docker"

      volume_mount {
        volume      = "prometheus"
        destination = "/var/lib/prometheus"
        read_only   = false
      }

      config {
        image = "prom/prometheus:latest"
        args = [
          "--config.file=/etc/prometheus/prometheus.yml",
          "--storage.tsdb.path=/var/lib/prometheus",
          "--storage.tsdb.max-block-duration=2h",
          "--storage.tsdb.min-block-duration=2h",
          "--web.external-url=/prometheus/",
          "--web.route-prefix=/prometheus/"
        ]

        volumes = [
          "local/prometheus.yml:/etc/prometheus/prometheus.yml",
        ]
      }
    }
  
    task "thanos-sidecar" {
      driver = "docker"

      volume_mount {
        volume      = "prometheus"
        destination = "/var/lib/prometheus"
        read_only   = false
      }
      
      template {
        change_mode = "noop"
        destination = "local/bucket.yml"

        data = <<EOH
type: S3
config:
  bucket: <your-bucket-name>
  endpoint: <your-bucket-endpoint>
  signature_version2: true
  access_key: <your-bucket-access-key>
  secret_key: <your-bucket-secret-key>
EOH
      }

      config {
        image = "quay.io/thanos/thanos:v0.17.2"
        args = [
          "sidecar",
          "--tsdb.path=/var/lib/prometheus",
          "--prometheus.url=http://localhost:9090/prometheus",
          "--grpc-address=0.0.0.0:10901",
          "--http-address=0.0.0.0:10902",
          "--objstore.config-file=/etc/thanos/bucket.yml"
        ]

        volumes = [
          "local/bucket.yml:/etc/thanos/bucket.yml",
        ]
      }

      resources {
        memory = 1024
      }
    }
  }

  group "thanos-store" {
    count = 1

    network {
      mode = "bridge"

      port "grpc" {
        to = 10901
      }
      port "http" {
        to = 10902
      }
    }

    service {
      name = "thanos-store"
      port = "grpc"
      tags = ["grpc"]
    }
    
    service {
      name = "thanos-store"
      port = "http"
      tags = ["http"]
    }

    task "thanos-store" {
      driver = "docker"

      template {
        change_mode = "noop"
        destination = "local/bucket.yml"

        data = <<EOH
type: S3
config:
  bucket: <your-bucket-name>
  endpoint: <your-bucket-endpoint>
  signature_version2: true
  access_key: <your-bucket-access-key>
  secret_key: <your-bucket-secret-key>
EOH
      }

      config {
        image = "quay.io/thanos/thanos:v0.17.2"
        args = [
          "store",
          "--grpc-address=0.0.0.0:10901",
          "--http-address=0.0.0.0:10902",
          "--objstore.config-file=/etc/thanos/bucket.yml"
        ]

        volumes = [
          "local/bucket.yml:/etc/thanos/bucket.yml",
        ]
      }
    }
  }

  group "thanos-query" {
    count = 1

    network {
      mode = "bridge"
      port "http" {
        to = 10902
      }
    }

    service {
      name = "thanos-query"
      port = "http"
      tags = ["urlprefix-/thanos"]

      check {
        name     = "thanos query port alive"
        type     = "http"
        path     = "/-/healthy"
        interval = "10s"
        timeout  = "2s"
      }
    }

    task "thanos-query" {
      driver = "docker"

      template {
        change_mode = "restart"
        destination = "local/targets.yml"

        data = <<EOH
- targets:{{ range service "thanos-sidecar" }}{{if .Tags | contains "grpc"}}
    - '{{ .Address }}:{{ .Port }}'{{ end }}{{ end }}
- targets:{{ range service "thanos-store" }}{{if .Tags | contains "grpc"}}
    - '{{ .Address }}:{{ .Port }}'{{ end }}{{ end }}
EOH
      }      

      config {
        image = "quay.io/thanos/thanos:v0.17.2"
        args = [
          "query",
          "--grpc-address=0.0.0.0:10901",
          "--http-address=0.0.0.0:10902",
          "--query.replica-label=prometheus_replica",
          "--query.replica-label=rule_replica",
          "--store.sd-files=/etc/targets.yml",
          "--web.external-prefix=/thanos/",
          "--web.route-prefix=/thanos/"          
        ]

        volumes = [
          "local/targets.yml:/etc/targets.yml",
        ]
      }
    }
  }

  group "thanos-compactor" {
    count = 1

    network {
      mode = "bridge"

      port "grpc" {
        to = 10901
      }
      port "http" {
        to = 10902
      }
    }

    service {
      name = "thanos-compactor"
      port = "grpc"
      tags = ["grpc"]
    }
    
    service {
      name = "thanos-compactor"
      port = "http"
      tags = ["http"]
    }

    task "thanos-compactor" {
      driver = "docker"

      template {
        change_mode = "noop"
        destination = "local/bucket.yml"

        data = <<EOH
type: S3
config:
  bucket: <your-bucket-name>
  endpoint: <your-bucket-endpoint>
  signature_version2: true
  access_key: <your-bucket-access-key>
  secret_key: <your-bucket-secret-key>
EOH
      }

      config {
        image = "quay.io/thanos/thanos:v0.17.2"
        args = [
          "compact",
          "--wait",
          "--objstore.config-file=/etc/thanos/bucket.yml"
        ]

        volumes = [
          "local/bucket.yml:/etc/thanos/bucket.yml",
        ]
      }
    }
  }
  
}