Add Pulsar support (#1204)

* Add pulsar connector

* Add unit tests for PulsarIODataset

* Fix lint errors

* Add PulsarWriter

* Add unit tests for PulsarWriter

* Add PulsarResourceBase to reduce repeated code

* Start Pulsar service for tests in Github Actions

* Fix CI error

* Reuse variables

* Fix undefined symbol BIO_f_base64

* Fix conflict of test environment between Pulsar and Kafka

* Increase timeout to avoid flaky

* Update Copyright

Author: BewareMyPower

.github/workflows/build.yml

@@ -206,6 +206,7 @@ jobs:
           bash -x -e tests/test_azure/start_azure.sh
           bash -x -e tests/test_pubsub/pubsub_test.sh
           bash -x -e tests/test_aws/aws_test.sh
+          bash -x -e tests/test_pulsar/pulsar_test.sh
       - name: Install ${{ matrix.python }} macOS
         run: |
           set -x -e
@@ -304,6 +305,7 @@ jobs:
           bash -x -e tests/test_azure/start_azure.sh
           bash -x -e tests/test_sql/sql_test.sh
           bash -x -e tests/test_gcloud/test_gcs.sh gcs-emulator
+          bash -x -e tests/test_pulsar/pulsar_test.sh
       - name: Test Linux
         run: |
           set -x -e

WORKSPACE

@@ -405,6 +405,9 @@ http_archive(
 
 http_archive(
     name = "boringssl",
+    patch_cmds = [
+        """sed -i.bak 's/bio.c",/bio.c","src\\/decrepit\\/bio\\/base64_bio.c",/g' BUILD.generated.bzl""",
+    ],
     sha256 = "1188e29000013ed6517168600fc35a010d58c5d321846d6a6dfee74e4c788b45",
     strip_prefix = "boringssl-7f634429a04abc48e2eb041c81c5235816c96514",
     urls = [
@@ -1063,3 +1066,29 @@ http_archive(
         "https://downloads.apache.org/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-src.tar.gz",
     ],
 )
+
+http_archive(
+    name = "dlfcn-win32",
+    build_file = "//third_party:dlfcn-win32.BUILD",
+    sha256 = "f18a412e84d8b701e61a78252411fe8c72587f52417c1ef21ca93604de1b9c55",
+    strip_prefix = "dlfcn-win32-1.2.0",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/dlfcn-win32/dlfcn-win32/archive/v1.2.0.tar.gz",
+        "https://github.com/dlfcn-win32/dlfcn-win32/archive/v1.2.0.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "pulsar",
+    build_file = "//third_party:pulsar.BUILD",
+    patch_cmds = [
+        "cp pulsar-common/src/main/proto/PulsarApi.proto pulsar-client-cpp/lib",
+        "sed -i.bak 's/define PULSAR_DEFINES_H_/define PULSAR_DEFINES_H_\\'$'\\n''#if defined(_MSC_VER)\\'$'\\n''#include <Windows.h>\\'$'\\n''#undef ERROR\\'$'\\n''#endif/g' pulsar-client-cpp/include/pulsar/defines.h",
+        "sed -i.bak 's/define LIB_ACKGROUPINGTRACKER_H_/define LIB_ACKGROUPINGTRACKER_H_\\'$'\\n''#include <pulsar\\/defines.h>/g' pulsar-client-cpp/lib/AckGroupingTracker.h",
+    ],
+    sha256 = "08f19ca6d6353751ff0661403b16b71425bf7ada3d8835a38e426ae303b0e385",
+    strip_prefix = "pulsar-2.6.1",
+    urls = [
+        "https://github.com/apache/pulsar/archive/v2.6.1.tar.gz",
+    ],
+)

tensorflow_io/core/BUILD

@@ -663,6 +663,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "pulsar_ops",
+    srcs = [
+        "kernels/pulsar_kernel.cc",
+        "ops/pulsar_ops.cc",
+    ],
+    copts = tf_io_copts(),
+    linkstatic = True,
+    deps = [
+        "@local_config_tf//:libtensorflow_framework",
+        "@local_config_tf//:tf_header_lib",
+        "@pulsar",
+    ],
+    alwayslink = 1,
+)
+
 cc_binary(
     name = "python/ops/libtensorflow_io.so",
     copts = tf_io_copts(),
@@ -684,6 +700,7 @@ cc_binary(
         "//tensorflow_io/core:numpy_ops",
         "//tensorflow_io/core:parquet_ops",
         "//tensorflow_io/core:pcap_ops",
+        "//tensorflow_io/core:pulsar_ops",
         "//tensorflow_io/core:operation_ops",
         "//tensorflow_io/core:pubsub_ops",
         "//tensorflow_io/core:serialization_ops",