Add tracing support for internals and JSON-RPC

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java

@@ -204,6 +204,10 @@ public void startNode(final BesuNode node) {
         params.add("--metrics-category");
         params.add(((Enum<?>) category).name());
       }
+      if (node.isMetricsEnabled() || metricsConfiguration.isPushEnabled()) {
+        params.add("--metrics-protocol");
+        params.add(metricsConfiguration.getProtocol().name());
+      }
       if (metricsConfiguration.isPushEnabled()) {
         params.add("--metrics-push-enabled");
         params.add("--metrics-push-host");

acceptance-tests/tests/build.gradle

@@ -24,13 +24,23 @@ dependencies {
   testImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts')
   testImplementation project(':ethereum:permissioning')
   testImplementation project(':ethereum:rlp')
+  testImplementation project(':metrics:core')
   testImplementation project(':plugin-api')
   testImplementation project(':privacy-contracts')
   testImplementation project(':testutil')
   testImplementation project(':util')
 
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone'
   testImplementation 'commons-io:commons-io'
+  testImplementation 'io.grpc:grpc-core'
+  testImplementation 'io.grpc:grpc-netty'
+  testImplementation 'io.grpc:grpc-stub'
+  testImplementation 'io.netty:netty-all'
+  testImplementation 'io.opentelemetry:opentelemetry-api'
+  testImplementation 'io.opentelemetry:opentelemetry-proto'
+  testImplementation 'io.opentelemetry:opentelemetry-sdk'
+  testImplementation 'io.opentelemetry:opentelemetry-sdk-trace'
+  testImplementation 'io.opentelemetry:opentelemetry-exporter-otlp'
   testImplementation 'junit:junit'
   testImplementation 'net.consensys:orion'
   testImplementation 'org.apache.commons:commons-compress'

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/OpenTelemetryAcceptanceTest.java

@@ -0,0 +1,171 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance;
+
+import static java.util.Collections.singletonList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.metrics.MetricsProtocol;
+import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
+import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase;
+import org.hyperledger.besu.tests.acceptance.dsl.WaitUtils;
+import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode;
+import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.BesuNodeConfigurationBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.io.Closer;
+import io.grpc.Server;
+import io.grpc.Status;
+import io.grpc.netty.NettyServerBuilder;
+import io.grpc.stub.StreamObserver;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse;
+import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse;
+import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
+import io.opentelemetry.proto.trace.v1.ResourceSpans;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class OpenTelemetryAcceptanceTest extends AcceptanceTestBase {
+
+  private static final class FakeCollector extends TraceServiceGrpc.TraceServiceImplBase {
+    private final List<ResourceSpans> receivedSpans = new ArrayList<>();
+    private Status returnedStatus = Status.OK;
+
+    @Override
+    public void export(
+        final ExportTraceServiceRequest request,
+        final StreamObserver<ExportTraceServiceResponse> responseObserver) {
+      receivedSpans.addAll(request.getResourceSpansList());
+      responseObserver.onNext(ExportTraceServiceResponse.newBuilder().build());
+      if (!returnedStatus.isOk()) {
+        if (returnedStatus.getCode() == Status.Code.DEADLINE_EXCEEDED) {
+          // Do not call onCompleted to simulate a deadline exceeded.
+          return;
+        }
+        responseObserver.onError(returnedStatus.asRuntimeException());
+        return;
+      }
+      responseObserver.onCompleted();
+    }
+
+    List<ResourceSpans> getReceivedSpans() {
+      return receivedSpans;
+    }
+
+    void setReturnedStatus(final Status returnedStatus) {
+      this.returnedStatus = returnedStatus;
+    }
+  }
+
+  private static final class FakeMetricsCollector
+      extends MetricsServiceGrpc.MetricsServiceImplBase {
+    private final List<ResourceMetrics> receivedMetrics = new ArrayList<>();
+    private Status returnedStatus = Status.OK;
+
+    @Override
+    public void export(
+        final ExportMetricsServiceRequest request,
+        final StreamObserver<ExportMetricsServiceResponse> responseObserver) {
+
+      receivedMetrics.addAll(request.getResourceMetricsList());
+      responseObserver.onNext(ExportMetricsServiceResponse.newBuilder().build());
+      if (!returnedStatus.isOk()) {
+        if (returnedStatus.getCode() == Status.Code.DEADLINE_EXCEEDED) {
+          // Do not call onCompleted to simulate a deadline exceeded.
+          return;
+        }
+        responseObserver.onError(returnedStatus.asRuntimeException());
+        return;
+      }
+      responseObserver.onCompleted();
+    }
+
+    List<ResourceMetrics> getReceivedMetrics() {
+      return receivedMetrics;
+    }
+
+    void setReturnedStatus(final Status returnedStatus) {
+      this.returnedStatus = returnedStatus;
+    }
+  }
+
+  private final FakeMetricsCollector fakeMetricsCollector = new FakeMetricsCollector();
+  private final FakeCollector fakeTracesCollector = new FakeCollector();
+  private final Closer closer = Closer.create();
+
+  private BesuNode metricsNode;
+
+  @Before
+  public void setUp() throws Exception {
+    Server server =
+        NettyServerBuilder.forPort(4317)
+            .addService(fakeTracesCollector)
+            .addService(fakeMetricsCollector)
+            .build()
+            .start();
+    closer.register(server::shutdownNow);
+
+    MetricsConfiguration configuration =
+        MetricsConfiguration.builder()
+            .protocol(MetricsProtocol.OPENTELEMETRY)
+            .enabled(true)
+            .port(0)
+            .hostsAllowlist(singletonList("*"))
+            .build();
+    metricsNode =
+        besu.create(
+            new BesuNodeConfigurationBuilder()
+                .name("metrics-node")
+                .jsonRpcEnabled()
+                .metricsConfiguration(configuration)
+                .build());
+    cluster.start(metricsNode);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    closer.close();
+  }
+
+  @Test
+  public void metricsReporting() {
+    WaitUtils.waitFor(
+        30,
+        () -> {
+          List<ResourceMetrics> resourceMetrics = fakeMetricsCollector.getReceivedMetrics();
+          assertThat(resourceMetrics.isEmpty()).isFalse();
+        });
+  }
+
+  @Test
+  public void traceReporting() {
+
+    WaitUtils.waitFor(
+        30,
+        () -> {
+          // call the json RPC endpoint to generate a trace.
+          net.netVersion().verify(metricsNode);
+          List<ResourceSpans> spans = fakeTracesCollector.getReceivedSpans();
+          assertThat(spans.isEmpty()).isFalse();
+        });
+  }
+}

build.gradle

@@ -489,8 +489,6 @@ applicationDefaultJvmArgs = [
 run {
   args project.hasProperty("besu.run.args") ? project.property("besu.run.args").toString().split("\\s+") : []
   doFirst {
-    applicationDefaultJvmArgs.add(0, "-Dotel.resource.attributes=service.name=besu-dev")
-    applicationDefaultJvmArgs.add(0, "-javaagent:"+ configurations.javaAgent.singleFile.toPath() + "")
     applicationDefaultJvmArgs = applicationDefaultJvmArgs.collect {
       it.replace('BESU_HOME', "$buildDir/besu")
     }
@@ -504,19 +502,6 @@ def tweakStartScript(createScriptTask) {
 
   createScriptTask.unixScript.text = createScriptTask.unixScript.text.replace('BESU_HOME', '\$APP_HOME')
   createScriptTask.windowsScript.text = createScriptTask.windowsScript.text.replace('BESU_HOME', '%~dp0..')
-  // OpenTelemetry Wiring for unix scripts
-  def agentFileName = configurations.javaAgent.singleFile.toPath().getFileName()
-  def unixRegex = $/exec "$$JAVACMD" /$
-  def forwardSlash = "/"
-  def unixReplacement = $/if [ -n "$$TRACING" ];then
- TRACING_AGENT="-javaagent:$$APP_HOME/agent${forwardSlash}${agentFileName}"
-fi
-exec "$$JAVACMD" $$TRACING_AGENT /$
-  createScriptTask.unixScript.text = createScriptTask.unixScript.text.replace(unixRegex, unixReplacement)
-  // OpenTelemetry Wiring for windows scripts
-  def windowsRegex = $/"%JAVA_EXE%" %DEFAULT_JVM_OPTS%/$
-  def windowsReplacement = $/if Defined TRACING (TRACING_AGENT="-javaagent:" "%APP_HOME%\agent\/$ + agentFileName + '")\r\n"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %TRACING_AGENT%'
-  createScriptTask.windowsScript.text = createScriptTask.windowsScript.text.replace(windowsRegex, windowsReplacement)
 
   // Prevent the error originating from the 8191 chars limit on Windows
   createScriptTask.windowsScript.text =
@@ -790,14 +775,9 @@ tasks.register("verifyDistributions") {
   }
 }
 
-configurations {
-  javaAgent
-}
-
 dependencies {
   implementation project(':besu')
   implementation project(':ethereum:evmtool')
-  javaAgent group: 'io.opentelemetry.instrumentation.auto', name: 'opentelemetry-javaagent', classifier: 'all'
   errorprone 'com.google.errorprone:error_prone_core'
 }
 
@@ -808,7 +788,6 @@ distributions {
       from("build/reports/license/license-dependency.html") { into "." }
       from("./docs/GettingStartedBinaries.md") { into "." }
       from("./docs/DocsArchive0.8.0.html") { into "." }
-      from(configurations.javaAgent.singleFile.toPath()) { into "agent"}
     }
   }
 }

docker/graalvm/Dockerfile

@@ -21,11 +21,7 @@ ENV BESU_RPC_HTTP_HOST 0.0.0.0
 ENV BESU_RPC_WS_HOST 0.0.0.0
 ENV BESU_GRAPHQL_HTTP_HOST 0.0.0.0
 ENV BESU_PID_PATH "/tmp/pid"
-# Tracing defaults
-# To enable tracing, uncomment next line
-#ENV TRACING=ENABLED
-ENV OTEL_EXPORTER=otlp
-ENV OTEL_OTLP_ENDPOINT="0.0.0.0:55680"
+
 ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu-$VERSION"
 
 ENV PATH="/opt/besu/bin:${PATH}"

docker/openjdk-11/Dockerfile

@@ -21,11 +21,7 @@ ENV BESU_RPC_HTTP_HOST 0.0.0.0
 ENV BESU_RPC_WS_HOST 0.0.0.0
 ENV BESU_GRAPHQL_HTTP_HOST 0.0.0.0
 ENV BESU_PID_PATH "/tmp/pid"
-# Tracing defaults
-# To enable tracing, uncomment next line
-#ENV TRACING=ENABLED
-ENV OTEL_EXPORTER=otlp
-ENV OTEL_OTLP_ENDPOINT="0.0.0.0:55680"
+
 ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu-$VERSION"
 
 ENV PATH="/opt/besu/bin:${PATH}"

docker/openjdk-latest/Dockerfile

@@ -21,11 +21,7 @@ ENV BESU_RPC_HTTP_HOST 0.0.0.0
 ENV BESU_RPC_WS_HOST 0.0.0.0
 ENV BESU_GRAPHQL_HTTP_HOST 0.0.0.0
 ENV BESU_PID_PATH "/tmp/pid"
-# Tracing defaults
-# To enable tracing, uncomment next line
-#ENV TRACING=ENABLED
-ENV OTEL_EXPORTER=otlp
-ENV OTEL_OTLP_ENDPOINT="0.0.0.0:55680"
+
 ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu-$VERSION"
 
 ENV PATH="/opt/besu/bin:${PATH}"

docs/tracing/README.md

@@ -10,7 +10,7 @@ To try out this example, start the Open Telemetry Collector and the Zipkin servi
 
 Start besu with:
 
-`$> ./gradlew run --args="--network=dev --rpc-http-enabled"`
+`$> OTEL_RESOURCE_ATTRIBUTES="service.name=besu-dev" OTEL_EXPORTER_OTLP_METRIC_INSECURE=true OTEL_EXPORTER_OTLP_SPAN_INSECURE=true ./gradlew run --args="--network=dev --rpc-http-enabled --metrics-enabled --metrics-protocol=opentelemetry"`
 
 Try interacting with the JSON-RPC API. Here is a simple example using cURL:
 
@@ -19,3 +19,6 @@ Try interacting with the JSON-RPC API. Here is a simple example using cURL:
 Open the Zipkin UI by browsing to http://localhost:9411/
 
 You will be able to see the detail of your traces.
+
+References:
+* [OpenTelemetry Environment Variable Specification](https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/sdk-environment-variables.md)

docs/tracing/docker-compose.yml

@@ -1,7 +1,8 @@
 version: "3"
 services:
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.11.0
+  otelcollector:
+    image: otel/opentelemetry-collector-contrib:0.17.0
+    container_name: otelcollector
     command: ["--config=/etc/otel-collector-config.yml"]
     volumes:
       - ./otel-collector-config.yml:/etc/otel-collector-config.yml
@@ -10,13 +11,20 @@ services:
       - "8888:8888"   # Prometheus metrics exposed by the collector
       - "8889:8889"   # Prometheus exporter metrics
       - "13133:13133" # health_check extension
-      - "55680:55680" # zpages extension
+      - "4317:4317"   # OTLP GRPC receiver
     depends_on:
       - zipkin
+      - metricsviewer
 
-  #Zipkin
+  # Zipkin
   zipkin:
     image: openzipkin/zipkin
     container_name: zipkin
     ports:
       - 9411:9411
+
+  metricsviewer:
+    image: docker.io/tmio/metrics-ui
+    container_name: metricsviewer
+    ports:
+      - 8080:8080

docs/tracing/otel-collector-config.yml

@@ -2,11 +2,17 @@ receivers:
   otlp:
     protocols:
       grpc:
-      http:
 
 exporters:
   zipkin:
     endpoint: "http://zipkin:9411/api/v2/spans"
+  otlp:
+    endpoint: "metricsviewer:4317"
+    insecure: true
+  logging:
+    loglevel: debug
+    sampling_initial: 5
+    sampling_thereafter: 200
 
 processors:
   batch:
@@ -23,3 +29,7 @@ service:
       receivers: [otlp]
       exporters: [zipkin]
       processors: [batch]
+    metrics:
+      receivers: [otlp]
+      exporters: [otlp, logging]
+      processors: [batch]

ethereum/api/build.gradle

@@ -49,6 +49,8 @@ dependencies {
 
   implementation 'com.google.guava:guava'
   implementation 'com.graphql-java:graphql-java'
+  implementation 'io.opentelemetry:opentelemetry-api'
+  implementation 'io.opentelemetry:opentelemetry-extension-trace-propagators'
   implementation 'io.vertx:vertx-auth-jwt'
   implementation 'io.vertx:vertx-core'
   implementation 'io.vertx:vertx-unit'