From dea74c5f8afcc91d4044030d501be6983cc7da7a Mon Sep 17 00:00:00 2001
From: Ankit Nayan <ankit@signoz.io>
Date: Wed, 20 Oct 2021 09:28:47 +0530
Subject: [PATCH] Span metrics generation (#347)

* feat: release for span to metrics

* feat: release for span to metrics
---
 .../clickhouse-setup/docker-compose.yaml      | 16 +++++--
 .../otel-collector-config.yaml                | 19 ++++++--
 .../otel-collector-metrics-config.yaml        | 44 +++++++++++++++++++
 3 files changed, 71 insertions(+), 8 deletions(-)
 create mode 100644 deploy/docker/clickhouse-setup/otel-collector-metrics-config.yaml

diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml
index e962067de0c..47891374677 100644
--- a/deploy/docker/clickhouse-setup/docker-compose.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose.yaml
@@ -22,7 +22,7 @@ services:
         retries: 3
 
   query-service:
-    image: signoz/query-service:0.4.1
+    image: signoz/query-service:0.4.2
     container_name: query-service
     command: ["-config=/root/config/prometheus.yml"]
     ports:
@@ -43,7 +43,7 @@ services:
         condition: service_healthy
     
   frontend:
-    image: signoz/frontend:0.4.1
+    image: signoz/frontend:0.4.2
     container_name: frontend
 
     depends_on:
@@ -57,7 +57,7 @@ services:
 
 
   otel-collector:
-    image: signoz/otelcontribcol:0.4.0
+    image: signoz/otelcontribcol:0.4.2
     command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"]
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@@ -71,11 +71,19 @@ services:
       - "4317:4317"       # OTLP GRPC receiver
       - "55679:55679" # zpages extension
       - "13133"       # health_check
-
+      - "8889:8889"   # prometheus exporter
     depends_on:
       clickhouse:
         condition: service_healthy
 
+  otel-collector-metrics:
+    image: signoz/otelcontribcol:0.4.2
+    command: ["--config=/etc/otel-collector-metrics-config.yaml", "--mem-ballast-size-mib=683"]
+    volumes:
+      - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
+    depends_on:
+      clickhouse:
+        condition: service_healthy
   hotrod:
     image: jaegertracing/example-hotrod:latest
     container_name: hotrod
diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml
index e2365634df4..eeaf7221d72 100644
--- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml
+++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml
@@ -1,4 +1,8 @@
 receivers:
+  otlp/spanmetrics:
+    protocols:
+      grpc:
+        endpoint: "localhost:12345"
   otlp:
     protocols:
       grpc:
@@ -8,7 +12,7 @@ receivers:
       grpc:
       thrift_http:
   hostmetrics:
-    collection_interval: 10s
+    collection_interval: 30s
     scrapers:
       cpu:
       load:
@@ -20,6 +24,9 @@ processors:
   batch:
     send_batch_size: 1000
     timeout: 10s
+  signozspanmetrics/prometheus:
+    metrics_exporter: prometheus
+    latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ]
   memory_limiter:
     # Same as --mem-ballast-size-mib CLI argument
     ballast_size_mib: 683
@@ -42,15 +49,19 @@ exporters:
     endpoint: tcp://clickhouse:9000/?database=signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
-
+  prometheus:
+    endpoint: "0.0.0.0:8889"
 service:
   extensions: [health_check, zpages]
   pipelines:
     traces:
       receivers: [jaeger, otlp]
-      processors: [batch]
+      processors: [signozspanmetrics/prometheus, batch]
       exporters: [clickhouse]
     metrics:
       receivers: [otlp, hostmetrics]
       processors: [batch]
-      exporters: [clickhousemetricswrite]
\ No newline at end of file
+      exporters: [clickhousemetricswrite]
+    metrics/spanmetrics:
+      receivers: [otlp/spanmetrics]
+      exporters: [prometheus]
\ No newline at end of file
diff --git a/deploy/docker/clickhouse-setup/otel-collector-metrics-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-metrics-config.yaml
new file mode 100644
index 00000000000..c6d63524d99
--- /dev/null
+++ b/deploy/docker/clickhouse-setup/otel-collector-metrics-config.yaml
@@ -0,0 +1,44 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+  # Data sources: metrics
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: "otel-collector"
+          scrape_interval: 60s
+          static_configs:
+            - targets: ["otel-collector:8889"]
+processors:
+  batch:
+    send_batch_size: 1000
+    timeout: 10s
+  memory_limiter:
+    # Same as --mem-ballast-size-mib CLI argument
+    ballast_size_mib: 683
+    # 80% of maximum memory up to 2G
+    limit_mib: 1500
+    # 25% of limit up to 2G
+    spike_limit_mib: 512
+    check_interval: 5s
+  # queued_retry:
+  #   num_workers: 4
+  #   queue_size: 100
+  #   retry_on_failure: true
+extensions:
+  health_check: {}
+  zpages: {}
+exporters:
+  clickhousemetricswrite:
+    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
+
+service:
+  extensions: [health_check, zpages]
+  pipelines:
+    metrics:
+      receivers: [otlp, prometheus]
+      processors: [batch]
+      exporters: [clickhousemetricswrite]
\ No newline at end of file