a custom app for grafana alloy that i use on truenas scale

alloy.yaml

configs:
  alloy_config:
    content: |
      // ===== SYSTEM METRICS EXPORT =====
      prometheus.exporter.unix "integrations_node_exporter" {
        procfs_path          = "/host/proc"
        sysfs_path           = "/host/sys"
        include_exporter_metrics = true
        enable_collectors        = ["systemd"]
        disable_collectors       = ["mdadm"]
      }

      discovery.relabel "integrations_node_exporter" {
        targets = prometheus.exporter.unix.integrations_node_exporter.targets

        rule {
          target_label = "job"
          replacement  = "integrations/node_exporter"
        }

        rule {
          replacement  = env("INSTANCE_NAME")
          target_label = "instance"
        }
      }

      prometheus.scrape "integrations_node_exporter" {
        targets         = discovery.relabel.integrations_node_exporter.output
        forward_to      = [otelcol.receiver.prometheus.metrics.receiver]
        job_name        = "integrations/node_exporter"
        scrape_interval = "15s"
      }

      // ===== CADVISOR (DOCKER CONTAINER METRICS) =====
      prometheus.exporter.cadvisor "docker" {
        docker_host      = "unix:///var/run/docker.sock"
        storage_duration = "5m"
      }

      discovery.relabel "cadvisor" {
        targets = prometheus.exporter.cadvisor.docker.targets

        rule {
          replacement  = env("INSTANCE_NAME")
          target_label = "instance"
        }
      }

      prometheus.scrape "cadvisor" {
        targets         = discovery.relabel.cadvisor.output
        forward_to      = [prometheus.relabel.cadvisor_fix_instance.receiver]
        job_name        = "integrations/cadvisor"
        scrape_interval = "15s"
      }

      // ZUSÄTZLICHER RELABEL STEP für cAdvisor
      prometheus.relabel "cadvisor_fix_instance" {
        forward_to = [otelcol.receiver.prometheus.metrics.receiver]

        // instance Label überschreiben falls vorhanden
        rule {
          replacement  = env("INSTANCE_NAME")
          target_label = "instance"
        }
      }

      // ===== ALLOY SELF METRICS =====
      prometheus.exporter.self "integrations_agent" { }

      discovery.relabel "integrations_agent" {
        targets = prometheus.exporter.self.integrations_agent.targets

        rule {
          replacement  = env("INSTANCE_NAME")
          target_label = "instance"
        }

        rule {
          target_label = "job"
          replacement  = "integrations/alloy"
        }
      }

      prometheus.scrape "integrations_agent" {
        targets         = discovery.relabel.integrations_agent.output
        forward_to      = [otelcol.receiver.prometheus.metrics.receiver]
        job_name        = "integrations/alloy"
        scrape_interval = "15s"
      }

      prometheus.scrape "smartctl" {
        targets = [
          {"__address__" = "192.168.0.209:9633", "instance"="storage"},
        ]
        forward_to = [otelcol.receiver.prometheus.metrics.receiver]
        scrape_interval = "15s"
        job_name = "smartctl"
      }

      // ===== OTEL RECEIVER FÜR METRICS =====
      otelcol.receiver.prometheus "metrics" {
        output {
          metrics = [otelcol.processor.attributes.metrics.input]
        }
      }

      otelcol.processor.attributes "metrics" {
        action {
          key    = "cloud.region"
          value  = env("CLOUD_REGION")
          action = "upsert"
        }

        action {
          key    = "cluster"
          value  = env("CLUSTER_NAME")
          action = "upsert"
        }

        // WICHTIG: instance nochmal forcieren
        action {
          key    = "instance"
          value  = env("INSTANCE_NAME")
          action = "upsert"
        }

        output {
          metrics = [otelcol.processor.batch.default.input]
        }
      }

      // ===== SYSTEM LOGS (JOURNAL) =====
      discovery.relabel "journal" {
        targets = []

        rule {
          source_labels = ["__journal__systemd_unit"]
          target_label  = "service_name"
          regex         = "(.+)\\.service"
          replacement   = "$1"
        }

        rule {
          source_labels = ["__journal_priority_keyword"]
          target_label  = "level"
        }

        rule {
          source_labels = ["__journal__transport"]
          target_label  = "transport"
        }

        rule {
          replacement  = env("INSTANCE_NAME")
          target_label = "instance"
        }
      }

      loki.source.journal "journal" {
        max_age       = "12h0m0s"
        relabel_rules = discovery.relabel.journal.rules
        forward_to    = [loki.process.journal.receiver]
        labels        = {
          job = "systemd-journal",
        }
      }

      loki.process "journal" {
        forward_to = [loki.write.logs_default.receiver]

        stage.match {
          selector = "{service_name=\"alloy\"}"

          stage.logfmt {
            mapping = {
              timestamp = "ts",
              log_level = "level",
              message   = "msg",
              service   = "service",
            }
          }

          stage.timestamp {
            source = "timestamp"
            format = "RFC3339Nano"
          }

          stage.label_drop {
            values = ["job"]
          }

          stage.static_labels {
            values = {
              job = "integrations/alloy",
            }
          }

          stage.labels {
            values = {
              level = "log_level",
            }
          }
        }

        stage.match {
          selector = "{transport=\"kernel\"}"

          stage.static_labels {
            values = {
              service_name = "kernel",
            }
          }
        }
      }

      // ===== DOCKER CONTAINER LOGS =====
      discovery.docker "containers" {
        host = "unix:///var/run/docker.sock"
      }

      loki.source.docker "containers" {
        host       = "unix:///var/run/docker.sock"
        targets    = discovery.docker.containers.targets
        forward_to = [loki.process.docker_logs.receiver]
      }

      loki.process "docker_logs" {
        forward_to = [loki.write.logs_default.receiver]
      }

      // ===== LOKI WRITE =====
      loki.write "logs_default" {
        endpoint {
          url = env("LOKI_ENDPOINT")
          headers = {
            "X-Scope-OrgID" = "langerma",
          }
        }
        external_labels = {
          cloud_region = env("CLOUD_REGION"),
          cluster      = env("CLUSTER_NAME"),
          instance     = env("INSTANCE_NAME"),
        }
      }

      // ===== TRACING =====
      tracing {
        sampling_fraction = 0.1
        write_to          = [otelcol.processor.batch.default.input]
      }

      // ===== BATCH PROCESSOR =====
      otelcol.processor.batch "default" {
        send_batch_size     = 1000
        send_batch_max_size = 1000
        timeout             = "200ms"

        output {
          traces  = [otelcol.exporter.otlphttp.cloud_traces.input]
          metrics = [otelcol.exporter.otlphttp.cloud_metrics.input]
          logs    = [otelcol.exporter.otlphttp.cloud_logs.input]
        }
      }

      // ===== OTLP EXPORTERS =====
      otelcol.exporter.otlphttp "cloud_traces" {
        client {
          endpoint = env("TEMPO_ENDPOINT")
          headers = {
            "X-Scope-OrgID" = "langerma",
          }
        }

        sending_queue {
          num_consumers = 30
          queue_size    = 1000
        }
      }

      otelcol.exporter.otlphttp "cloud_metrics" {
        client {
          endpoint = env("MIMIR_ENDPOINT")
          headers = {
            "X-Scope-OrgID" = "langerma",
          }
        }

        sending_queue {
          num_consumers = 30
          queue_size    = 1000
        }
      }

      otelcol.exporter.otlphttp "cloud_logs" {
        client {
          endpoint = env("LOKI_OTLP_ENDPOINT")
          headers = {
            "X-Scope-OrgID" = "langerma",
          }
        }

        sending_queue {
          num_consumers = 30
          queue_size    = 1000
        }
      }

      // ===== OTLP RECEIVER =====
      otelcol.receiver.otlp "local" {
        grpc {
          endpoint = "0.0.0.0:4317"
        }

        http {
          endpoint = "0.0.0.0:4318"
        }

        output {
          metrics = [otelcol.processor.batch.default.input]
          logs    = [otelcol.processor.batch.default.input]
          traces  = [otelcol.processor.batch.default.input]
        }
      }
services:
  alloy:
    command:
      - run
      - /etc/alloy/config.alloy
      - '--server.http.listen-addr=0.0.0.0:12345'
      - '--storage.path=/var/lib/alloy/data'
    configs:
      - source: alloy_config
        target: /etc/alloy/config.alloy
    container_name: alloy
    environment:
      CLOUD_REGION: somewhere
      CLUSTER_NAME: somecluster
      INSTANCE_NAME: somename
      LOKI_ENDPOINT: https://someloki/loki/api/v1/push
      LOKI_OTLP_ENDPOINT: https://someloki/otlp
      MIMIR_ENDPOINT: https://somemimir/otlp
      NVIDIA_VISIBLE_DEVICES: void
      TEMPO_ENDPOINT: https://sometempo/otlp
      TZ: Europe/Vienna
    group_add:
      - '568'
    image: grafana/alloy:latest
    network_mode: bridge
    platform: linux/amd64
    ports:
      - '12345:12345'
      - '4317:4317'
      - '4318:4318'
    privileged: True
    pull_policy: always
    restart: unless-stopped
    stdin_open: False
    tty: False
    user: '0:0'
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/host/root:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - alloy-data:/var/lib/alloy/data
version: '3.8'
volumes:
  alloy-data: Null
x-portals:
  - host: 0.0.0.0
    name: Web UI
    path: /
    port: 12345
    scheme: http