Altinity ClickHouse Operator Integration

Overview

ClickHouse exposes metrics through a Prometheus endpoint that you can scrape. We recommend to scrape and forward these metrics via an OpenTelemetry Collector.

Prometheus metrics endpoint

The clickhouse metrics and clickhouse operator's metrics are exposed via clickhouse operator's HTTP endpoints. There are 2 endpoints, one is for ClickHouse metrics, which is exposed via a standard Prometheus endpoint.

plaintext

12
prometheus.io/port: "8888"
prometheus.io/scrape: "true"

The other one is for metrics for the clickhouser operator it's self.

plaintext

12
clickhouse-operator-metrics/port: "9999"
clickhouse-operator-metrics/scrape: "true"

Collector Configuration

The following code snippet shows an OpenTelemetry Collector configuration file.

yaml

123456789101112131516171820212224252627293031323335363739404143444547484951525455575859606263646566676869707273747577787981828384858788899091939495979899101102103105106107109110112113115116118119120121122123125126127128129130
receivers:
  prometheus/clickhouse:
    config:
      scrape_configs:
        - job_name: "kubernetes-pods"
          kubernetes_sd_configs:
            - role: pod
          scrape_interval: 30s
          scrape_timeout: 5s
          tls_config:
            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
            insecure_skip_verify: false
          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
          relabel_configs:
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
              regex: true
              action: keep
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
              regex: false
              action: drop
            - source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_prometheus_io_port]
              regex: (.+);(.+)
              replacement: $1:$2
              target_label: __address__
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
              action: replace
              regex: ()
              replacement: /metrics
              target_label: __meta_kubernetes_pod_annotation_prometheus_io_path
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
              action: replace
              target_label: __metrics_path__
            - source_labels: [__meta_kubernetes_namespace]
              action: replace
              target_label: namespace
            - source_labels: [__meta_kubernetes_pod_name]
              action: replace
              target_label: pod_name
            - source_labels: [__meta_kubernetes_pod_container_name]
              action: replace
              target_label: container_name
            - action: labelmap
              regex: __meta_kubernetes_pod_label_(.+)
            - action: labeldrop
              regex: "container_id|image_id|controller_revision_hash|pod_template_generation|pod_template_hash"
          metric_relabel_configs:
            - source_labels: [__name__]
              regex: "process_virtual.*|process_max_fds|process_start_time.*|http_.*|go_gc.*"
              action: drop
        - job_name: "kubernetes-clickhouse-operator-pods"
          kubernetes_sd_configs:
            - role: pod
          scrape_interval: 30s
          scrape_timeout: 10s
          tls_config:
            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
            insecure_skip_verify: false
          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
          relabel_configs:
            - source_labels: [__meta_kubernetes_pod_annotation_clickhouse_operator_metrics_scrape]
              regex: true
              action: keep
            - source_labels: [__meta_kubernetes_pod_annotation_clickhouse_operator_metrics_scrape]
              regex: false
              action: drop
            - source_labels:
                [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_clickhouse_operator_metrics_port]
              regex: (.+);(.+)
              replacement: $1:$2
              target_label: __address__
            - source_labels: [__meta_kubernetes_pod_annotation_clickhouse_operator_metrics_path]
              action: replace
              regex: ()
              replacement: /metrics
              target_label: __meta_kubernetes_pod_annotation_clickhouse_operator_metrics_path
            - source_labels: [__meta_kubernetes_pod_annotation_clickhouse_operator_metrics_path]
              action: replace
              target_label: __metrics_path__
            - source_labels: [__meta_kubernetes_namespace]
              action: replace
              target_label: namespace
            - source_labels: [__meta_kubernetes_pod_name]
              action: replace
              target_label: pod_name
            - source_labels: [__meta_kubernetes_pod_container_name]
              action: replace
              target_label: container_name
            - action: labelmap
              regex: __meta_kubernetes_pod_label_(.+)
            - action: labeldrop
              regex: "container_id|image_id|controller_revision_hash|pod_template_generation|pod_template_hash"
processors:
  batch:
exporters:
  otlp/dash0:
    endpoint: "{{endpoint_otlp_grpc}}"
    headers:
      Authorization: "Bearer {{token}}"
      "Dash0-Dataset": "{{dataset}}"
service:
  pipelines:
    metrics:
      receivers: [prometheus/clickhouse]
      processors: [batch]
      exporters: [otlp/dash0]