Latest opentelemetry

.github/workflows/resources/unit-test-conf.yaml

@@ -37,6 +37,11 @@ spec:
   containers:
     - name: jasminegraph-container
       image: jasminegraph:unit
+      env:
+        - name: TEST_MODE
+          value: "true"
+        - name: DISABLE_TELEMETRY
+          value: "true"
       volumeMounts:
         - name: host-volume
           mountPath: /home/ubuntu/software/jasminegraph/coverage

CMakeLists.txt

@@ -8,10 +8,10 @@ if(NOT CMAKE_BUILD_TYPE)
     message(STATUS "Build type not specified, setting to RelWithDebInfo for VTune profiling")
 endif()
 
-
-
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+find_package(CURL REQUIRED)
 
 set(HEADERS globals.h
         src/backend/JasmineGraphBackend.h
@@ -91,6 +91,7 @@ set(HEADERS globals.h
         src/util/hdfs/HDFSStreamHandler.h
         src/partitioner/stream/HDFSMultiThreadedHashPartitioner.h
         src/query/processor/cypher/util/Const.h
+        src/util/telemetry/OpenTelemetryUtil.h
         src/nativestore/MetaPropertyLink.h
         src/nativestore/MetaPropertyEdgeLink.h
         src/query/processor/cypher/queryplanner/Operators.h
@@ -160,6 +161,8 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp
         src/util/kafka/InstanceStreamHandler.cpp
         src/util/logger/Logger.cpp
         src/util/scheduler/SchedulerService.cpp
+        src/util/telemetry/OpenTelemetryUtil.cpp
+        src/util/telemetry/TelemetryInitializer.cpp
         src/k8s/K8sInterface.cpp
         src/nativestore/NodeManager.cpp
         src/nativestore/NodeBlock.cpp
@@ -284,8 +287,8 @@ target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libflatb
 target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libjsoncpp.so)
 target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libcppkafka.so)
 target_link_libraries(JasmineGraphLib PRIVATE /usr/local/libhdfs3/lib/libhdfs3.so)
-target_link_libraries(JasmineGraph JasmineGraphLib)
-target_link_libraries(JasmineGraph curl)
+target_link_libraries(JasmineGraph PRIVATE JasmineGraphLib)
+target_link_libraries(JasmineGraph PRIVATE curl)
 
 include_directories(/usr/local/libhdfs3/include/hdfs/)
 include_directories(/usr/include/oneapi/)
@@ -300,6 +303,50 @@ target_link_libraries(JasmineGraphLib PRIVATE yaml-cpp)
 target_link_libraries(JasmineGraphLib PRIVATE curl)
 target_link_libraries(JasmineGraphLib PRIVATE antlr4-runtime)
 
+# Option to enable/disable OpenTelemetry
+option(ENABLE_OPENTELEMETRY "Enable OpenTelemetry support" ON)
+
+# Disable OpenTelemetry for DEBUG builds to avoid issues in unit tests
+if(CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    set(ENABLE_OPENTELEMETRY OFF)
+    message(STATUS "DEBUG build detected - disabling OpenTelemetry to prevent test issues")
+endif()
+
+# Conditionally find and link OpenTelemetry packages
+if(ENABLE_OPENTELEMETRY)
+    # Find OpenTelemetry packages
+    find_package(opentelemetry-cpp REQUIRED)
+    find_package(PkgConfig REQUIRED)
+
+    # Find nlohmann_json package for OpenTelemetry
+    find_package(nlohmann_json REQUIRED)
+
+    # OpenTelemetry libraries for native tracing support (without gRPC)
+    target_link_libraries(JasmineGraphLib PRIVATE opentelemetry-cpp::api)
+    target_link_libraries(JasmineGraphLib PRIVATE opentelemetry-cpp::sdk)
+    target_link_libraries(JasmineGraphLib PRIVATE opentelemetry-cpp::trace)
+    target_link_libraries(JasmineGraphLib PRIVATE opentelemetry-cpp::metrics)
+    target_link_libraries(JasmineGraphLib PRIVATE opentelemetry-cpp::ostream_span_exporter)
+    target_link_libraries(JasmineGraphLib PRIVATE opentelemetry-cpp::otlp_http_exporter)
+    target_link_libraries(JasmineGraphLib PRIVATE nlohmann_json::nlohmann_json)
+
+    # Link main executable with OpenTelemetry libraries
+    target_link_libraries(JasmineGraph PRIVATE opentelemetry-cpp::api)
+    target_link_libraries(JasmineGraph PRIVATE opentelemetry-cpp::sdk)
+    target_link_libraries(JasmineGraph PRIVATE opentelemetry-cpp::trace)
+    target_link_libraries(JasmineGraph PRIVATE opentelemetry-cpp::metrics)
+    target_link_libraries(JasmineGraph PRIVATE opentelemetry-cpp::ostream_span_exporter)
+    target_link_libraries(JasmineGraph PRIVATE opentelemetry-cpp::otlp_http_exporter)
+    target_link_libraries(JasmineGraph PRIVATE nlohmann_json::nlohmann_json)
+
+    message(STATUS "OpenTelemetry support enabled - linking OpenTelemetry libraries")
+else()
+    message(STATUS "OpenTelemetry support disabled - skipping OpenTelemetry library linking")
+endif()
+
+# Include OpenTelemetry headers
+target_include_directories(JasmineGraphLib PRIVATE /usr/local/include)
+
 
 if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
     # Include google test

Dockerfile

@@ -1,6 +1,6 @@
 FROM miyurud/jasminegraph-prerequisites:20250930T172442
 
-RUN apt-get update && apt-get install -y libcurl4-openssl-dev sysstat nmon
+RUN apt-get update && apt-get install -y libcurl4-openssl-dev sysstat nmon python3-setuptools
 RUN rm -r /usr/lib/python3.8/distutils
 RUN apt-get purge -y libpython3.8-dev python3.8-dev python3.8-distutils libpython3.8 python3.8
 

conf/prometheus.yaml

@@ -8,8 +8,7 @@ scrape_configs:
     static_configs:
       - targets: ["172.28.5.2:9090"]
 
-
   - job_name: "pushgateway"
     scrape_interval: 2s
     static_configs:
-      - targets: ["172.28.5.3:9091"]
+      - targets: ["172.28.5.3:9091"]

docker-compose.yml

@@ -21,12 +21,14 @@ services:
       - kafka
       - alloy
       - grafana
+      - tempo
   prometheus:
     image: prom/prometheus:latest
     ports:
       - 9090:9090
     volumes:
       - './conf/prometheus.yaml:/etc/prometheus/prometheus.yml'
+      - './prometheus_data:/prometheus' 
     networks:
       jasminenet:
         ipv4_address: 172.28.5.2
@@ -64,8 +66,6 @@ services:
     image: grafana/alloy:latest
     ports:
       - 12345:12345
-      - 4317:4317
-      - 4318:4318
     volumes:
       - ./config.alloy:/etc/alloy/config.alloy
       - /var/run/docker.sock:/var/run/docker.sock
@@ -111,11 +111,47 @@ services:
           isDefault: true
           version: 1
           editable: true 
+        - name: Tempo
+          type: tempo
+          access: proxy
+          orgId: 1
+          url: 'http://tempo:3200'
+          basicAuth: false
+          isDefault: false
+          version: 1
+          editable: true
+          jsonData:
+            httpMethod: GET
+            serviceMap:
+              datasourceUid: 'prometheus'
+        - name: Prometheus
+          type: prometheus
+          access: proxy
+          orgId: 1
+          url: 'http://prometheus:9090'
+          basicAuth: false
+          isDefault: false
+          version: 1
+          editable: true
         EOF
         /run.sh
     networks:
       jasminenet:
         ipv4_address: 172.28.5.6
+
+  tempo:
+    image: grafana/tempo:latest
+    ports:
+      - "3200:3200"
+      - "4317:4317"
+      - "4318:4318"
+      - "14268:14268"
+    volumes:
+      - ./tempo-config.yaml:/etc/tempo.yaml
+    command: -config.file=/etc/tempo.yaml
+    networks:
+      jasminenet:
+        ipv4_address: 172.28.5.9
 
 networks:
   jasminenet:

main.cpp

@@ -16,13 +16,13 @@ limitations under the License.
 #include <unistd.h>
 
 #include <future>
-#include <iostream>
 
 #include "globals.h"
 #include "src/k8s/K8sWorkerController.h"
 #include "src/server/JasmineGraphInstance.h"
 #include "src/util/logger/Logger.h"
 #include "src/util/scheduler/SchedulerService.h"
+#include "src/util/telemetry/TelemetryInitializer.h"
 
 unsigned int microseconds = 10000000;
 JasmineGraphServer *server;
@@ -54,7 +54,10 @@ enum worker_mode_args {
     WORKER_ENABLE_NMON = 7
 };
 
-void fnExit3(void) { delete (server); }
+void fnExit3(void) {
+    shutdownOpenTelemetry();
+    delete (server);
+}
 
 int main(int argc, char *argv[]) {
     atexit(fnExit3);
@@ -65,7 +68,7 @@ int main(int argc, char *argv[]) {
             "<hostName> <serverPort> <serverDataPort> to start as worker");
         return -1;
     }
-    std::cout << argc << std::endl;
+    main_logger.info(std::to_string(argc));
 
     int mode = atoi(argv[args::MODE]);
     std::string JASMINEGRAPH_HOME = Utils::getJasmineGraphHome();
@@ -77,6 +80,9 @@ int main(int argc, char *argv[]) {
     thread schedulerThread(SchedulerService::startScheduler);
 
     if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_MASTER) {
+        // Initialize OpenTelemetry for master process
+        initializeOpenTelemetry();
+
         std::string masterIp = argv[master_mode_args::MASTER_IP];
         int numberOfWorkers = atoi(argv[master_mode_args::NUMBER_OF_WORKERS]);
         std::string workerIps = argv[master_mode_args::WORKER_IPS];
@@ -97,6 +103,9 @@ int main(int argc, char *argv[]) {
     } else if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_WORKER) {
         main_logger.info(to_string(argc));
 
+        // Initialize OpenTelemetry for worker process
+        initializeWorkerTelemetry();
+
         if (argc < 8) {
             main_logger.info(
                 "Need 7 arguments. Use <mode> 2 <hostName> <masterIP> <serverPort> <serverDataPort> <enable-nmon> to "
@@ -113,7 +122,7 @@ int main(int argc, char *argv[]) {
         int serverDataPort = atoi(argv[worker_mode_args::SERVER_DATA_PORT]);
         enableNmon = argv[worker_mode_args::WORKER_ENABLE_NMON];
 
-        std::cout << "In worker mode" << std::endl;
+        main_logger.info("In worker mode");
         instance = new JasmineGraphInstance();
         instance->start_running(hostName, masterHost, serverPort, serverDataPort, enableNmon);