Integrating monoscope with Kubernetes

1. Add the Helm Repository

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm repo update

2. Create a Secret for Your API Key

kubectl create secret generic monoscope-secrets \
  --from-literal=api-key=YOUR_API_KEY

3. Agent Collector (DaemonSet) — Logs and Node Metrics

Create values-agent.yaml:

mode: daemonset

image:
  # Pin a specific tag so chart upgrades don't silently roll a new collector.
  # Current as of April 2026 — bump after reviewing the release notes:
  # https://github.com/open-telemetry/opentelemetry-collector-releases/releases
  repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s
  tag: "0.149.0"

presets:
  logsCollection:
    enabled: true          # filelog — reads /var/log/pods on each node
  kubeletMetrics:
    enabled: true          # kubeletstats — local kubelet
  kubernetesAttributes:
    enabled: true          # k8s_attributes processor — enriches with pod/namespace metadata
  # clusterMetrics and kubernetesEvents intentionally OFF here —
  # they belong on the cluster collector (one replica only).

extraEnvs:
  - name: MONOSCOPE_API_KEY
    valueFrom:
      secretKeyRef:
        name: monoscope-secrets
        key: api-key

config:
  # health_check must be declared explicitly: when you override config:, the
  # chart's default service.extensions block is replaced, and the readiness
  # probe (port 13133) starts failing.
  extensions:
    health_check:
      endpoint: 0.0.0.0:13133

  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318

  processors:
    batch: {}
    memory_limiter:
      check_interval: 1s
      limit_mib: 4000
      spike_limit_mib: 800
    # Declared explicitly because overriding config.processors replaces the
    # block the kubernetesAttributes preset would have injected.
    k8s_attributes:
      auth_type: serviceAccount
      passthrough: false
      extract:
        metadata:
          - k8s.pod.name
          - k8s.pod.uid
          - k8s.deployment.name
          - k8s.namespace.name
          - k8s.node.name
          - k8s.container.name
    resource:
      attributes:
        - key: x-api-key
          value: ${env:MONOSCOPE_API_KEY}
          action: upsert

  exporters:
    # tls.insecure is safe here: the monoscope ingest endpoint terminates TLS at
    # its load balancer; the collector connects over a plaintext-tolerant L4 path.
    otlp_grpc:
      endpoint: "otelcol.monoscope.tech:4317"
      tls:
        insecure: true

  # IMPORTANT: when you override service.pipelines, you must list every receiver
  # and processor you want active — including the ones the presets configured
  # (filelog, kubeletstats, k8s_attributes). The chart merges receivers/processors
  # but does NOT merge pipeline arrays. Same applies to service.extensions: it's
  # replaced wholesale, so health_check must be re-declared here.
  service:
    extensions: [health_check]
    pipelines:
      traces:
        receivers: [otlp]
        processors: [k8s_attributes, memory_limiter, batch, resource]
        exporters: [otlp_grpc]
      metrics:
        receivers: [otlp, kubeletstats]
        processors: [k8s_attributes, memory_limiter, batch, resource]
        exporters: [otlp_grpc]
      logs:
        receivers: [otlp, filelog]
        processors: [k8s_attributes, memory_limiter, batch, resource]
        exporters: [otlp_grpc]

Install it:

helm install monoscope-agent open-telemetry/opentelemetry-collector \
  --values values-agent.yaml

4. Cluster Collector (Deployment) — Events and Cluster Metrics

Create values-cluster.yaml:

mode: deployment
# Must stay 1 — k8s_events and k8s_cluster don't leader-elect, so >1 replica
# produces duplicate events and cluster metrics.
replicaCount: 1

image:
  repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s
  tag: "0.149.0"           # see release notes before bumping

presets:
  clusterMetrics:
    enabled: true          # k8s_cluster — cluster-wide metrics from API server
  kubernetesEvents:
    enabled: true          # k8s_events — emits events as logs
  kubernetesAttributes:
    enabled: true

extraEnvs:
  - name: MONOSCOPE_API_KEY
    valueFrom:
      secretKeyRef:
        name: monoscope-secrets
        key: api-key

config:
  # See agent values for why health_check must be re-declared here.
  extensions:
    health_check:
      endpoint: 0.0.0.0:13133

  # Receivers are declared explicitly so the pipelines below are unambiguous.
  # The presets generate matching RBAC; the receiver blocks are merged with
  # whatever the presets inject.
  receivers:
    k8s_cluster:
      collection_interval: 10s
    k8s_events:
      namespaces: []

  processors:
    batch: {}
    memory_limiter:
      check_interval: 1s
      limit_mib: 1000
      spike_limit_mib: 200
    # Must be declared explicitly — overriding config.processors replaces the
    # block the kubernetesAttributes preset would have injected, so the
    # pipelines below would otherwise fail with a "reference error" on startup.
    k8s_attributes:
      auth_type: serviceAccount
      passthrough: false
    resource:
      attributes:
        - key: x-api-key
          value: ${env:MONOSCOPE_API_KEY}
          action: upsert

  exporters:
    otlp_grpc:
      endpoint: "otelcol.monoscope.tech:4317"
      tls:
        insecure: true     # see note in agent values

  service:
    extensions: [health_check]
    pipelines:
      metrics:
        receivers: [k8s_cluster]
        processors: [k8s_attributes, memory_limiter, batch, resource]
        exporters: [otlp_grpc]
      logs:
        receivers: [k8s_events]
        processors: [k8s_attributes, memory_limiter, batch, resource]
        exporters: [otlp_grpc]

Install it:

helm install monoscope-cluster open-telemetry/opentelemetry-collector \
  --values values-cluster.yaml

5. Verify

kubectl get pods -l app.kubernetes.io/name=opentelemetry-collector
kubectl logs daemonset/monoscope-agent-opentelemetry-collector
kubectl logs deployment/monoscope-cluster-opentelemetry-collector

You should see the agent running on every node and a single cluster collector pod. Telemetry will start flowing to your monoscope dashboard shortly after the pods become Ready.

Install the Operator

The OTel Operator’s admission webhook requires cert-manager (or another certificate provider). Skip the first command if you already have cert-manager — or any compatible cert source — installed. For production, replace latest with a pinned release tag from each project.

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml
kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml

Create the same API key secret used by the Helm path (skip if already created):

kubectl create secret generic monoscope-secrets \
  --from-literal=api-key=YOUR_API_KEY

Agent CR (DaemonSet)

apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: monoscope-agent
spec:
  mode: daemonset
  volumeMounts:
    - name: varlogpods
      mountPath: /var/log/pods
      readOnly: true
  volumes:
    - name: varlogpods
      hostPath:
        path: /var/log/pods
  env:
    - name: K8S_NODE_NAME
      valueFrom:
        fieldRef:
          fieldPath: spec.nodeName
    - name: MONOSCOPE_API_KEY
      valueFrom:
        secretKeyRef:
          name: monoscope-secrets
          key: api-key
  config:
    extensions:
      health_check:
        endpoint: 0.0.0.0:13133
    receivers:
      filelog:
        # Path layout: /var/log/pods/<ns>_<pod>_<uid>/<container>/<restart>.log
        include: [/var/log/pods/*/*/*.log]
        start_at: end          # use 'beginning' only for first-install backfill
      kubeletstats:
        collection_interval: 10s
        auth_type: serviceAccount
        endpoint: ${env:K8S_NODE_NAME}:10250
        insecure_skip_verify: true   # acceptable for dev; in production mount the kubelet CA
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
    processors:
      batch: {}
      memory_limiter:
        check_interval: 1s
        limit_mib: 4000
        spike_limit_mib: 800
      k8s_attributes:
        auth_type: serviceAccount
        passthrough: false
        filter:
          node_from_env_var: K8S_NODE_NAME
      resource:
        attributes:
          - key: x-api-key
            value: ${env:MONOSCOPE_API_KEY}
            action: upsert
    exporters:
      otlp_grpc:
        endpoint: "otelcol.monoscope.tech:4317"
        tls:
          insecure: true
    service:
      extensions: [health_check]
      pipelines:
        traces:
          receivers: [otlp]
          processors: [k8s_attributes, memory_limiter, batch, resource]
          exporters: [otlp_grpc]
        metrics:
          receivers: [otlp, kubeletstats]
          processors: [k8s_attributes, memory_limiter, batch, resource]
          exporters: [otlp_grpc]
        logs:
          receivers: [filelog, otlp]
          processors: [k8s_attributes, memory_limiter, batch, resource]
          exporters: [otlp_grpc]

Cluster CR (Deployment, 1 replica)

apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: monoscope-cluster
spec:
  mode: deployment
  replicas: 1
  env:
    - name: MONOSCOPE_API_KEY
      valueFrom:
        secretKeyRef:
          name: monoscope-secrets
          key: api-key
  config:
    extensions:
      health_check:
        endpoint: 0.0.0.0:13133
    receivers:
      k8s_cluster:
        collection_interval: 10s
      k8s_events:
        namespaces: []
    processors:
      batch: {}
      memory_limiter:
        check_interval: 1s
        limit_mib: 1000
        spike_limit_mib: 200
      k8s_attributes:
        auth_type: serviceAccount
        passthrough: false
      resource:
        attributes:
          - key: x-api-key
            value: ${env:MONOSCOPE_API_KEY}
            action: upsert
    exporters:
      otlp_grpc:
        endpoint: "otelcol.monoscope.tech:4317"
        tls:
          insecure: true
    service:
      extensions: [health_check]
      pipelines:
        metrics:
          receivers: [k8s_cluster]
          processors: [k8s_attributes, memory_limiter, batch, resource]
          exporters: [otlp_grpc]
        logs:
          receivers: [k8s_events]
          processors: [k8s_attributes, memory_limiter, batch, resource]
          exporters: [otlp_grpc]

RBAC

Both CRs need read access to pods, namespaces, nodes, events, and the workload APIs. The OTel Operator creates a ServiceAccount named <cr-name>-collector in the CR’s namespace, so the binding below targets monoscope-agent-collector and monoscope-cluster-collector. Apply this once:

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: monoscope-collector
rules:
- apiGroups: [""]
  resources:
    - pods
    - namespaces
    - nodes
    - nodes/stats
    - nodes/proxy
    - services
    - events
    - replicationcontrollers
    - resourcequotas
    - persistentvolumes
    - persistentvolumeclaims
  verbs: [get, list, watch]
- apiGroups: ["apps"]
  resources: [deployments, replicasets, daemonsets, statefulsets]
  verbs: [get, list, watch]
- apiGroups: ["batch"]
  resources: [jobs, cronjobs]
  verbs: [get, list, watch]
- apiGroups: ["autoscaling"]
  resources: [horizontalpodautoscalers]
  verbs: [get, list, watch]
- apiGroups: ["events.k8s.io"]
  resources: [events]
  verbs: [get, list, watch]
- apiGroups: ["discovery.k8s.io"]
  resources: [endpointslices]
  verbs: [get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: monoscope-collector
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: monoscope-collector
subjects:
- kind: ServiceAccount
  name: monoscope-agent-collector
  namespace: default
- kind: ServiceAccount
  name: monoscope-cluster-collector
  namespace: default